From: Mark Beierl Date: Thu, 9 Feb 2023 20:01:33 +0000 (-0500) Subject: Merge remote-tracking branch 'origin/master' into paas X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=90f700db8e2287346b66cf3ed0b745a3375cfa38;p=osm%2FLCM.git Merge remote-tracking branch 'origin/master' into paas Change-Id: If911e353cbeb9f278c57c69812cbdfce0ad1e242 Signed-off-by: Mark Beierl --- 90f700db8e2287346b66cf3ed0b745a3375cfa38 diff --cc osm_lcm/nglcm.py index 67b2a3f,0000000..6972477 mode 100644,000000..100644 --- a/osm_lcm/nglcm.py +++ b/osm_lcm/nglcm.py @@@ -1,185 -1,0 +1,183 @@@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +## +# Copyright 2018 Telefonica S.A. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## + + +import asyncio +import getopt +import logging +import logging.handlers +import sys +import yaml + +from osm_lcm.lcm_utils import LcmException + +from osm_common.dbbase import DbException +from osm_lcm.data_utils.database.database import Database +from osm_lcm.data_utils.lcm_config import LcmCfg +from os import path +from temporalio import workflow +from temporalio.client import Client +from temporalio.worker import Worker + + +class NGLcm: - + main_config = LcmCfg() + + def __init__(self, config_file, loop=None): + """ + Init, Connect to database, filesystem storage, and messaging + :param config: two level dictionary with configuration. Top level should contain 'database', 'storage', + :return: None + """ + self.db = None + + # logging + self.logger = logging.getLogger("lcm") + # load configuration + config = self.read_config_file(config_file) + self.main_config.set_from_dict(config) + self.main_config.transform() + self.main_config.load_from_env() + self.logger.critical("Loaded configuration:" + str(self.main_config.to_dict())) + + # logging + log_format_simple = ( + "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s" + ) + log_formatter_simple = logging.Formatter( + log_format_simple, datefmt="%Y-%m-%dT%H:%M:%S" + ) + if self.main_config.globalConfig.logfile: + file_handler = logging.handlers.RotatingFileHandler( + self.main_config.globalConfig.logfile, + maxBytes=100e6, + backupCount=9, + delay=0, + ) + file_handler.setFormatter(log_formatter_simple) + self.logger.addHandler(file_handler) + if not self.main_config.globalConfig.to_dict()["nologging"]: + str_handler = logging.StreamHandler() + str_handler.setFormatter(log_formatter_simple) + self.logger.addHandler(str_handler) + + if self.main_config.globalConfig.to_dict()["loglevel"]: + self.logger.setLevel(self.main_config.globalConfig.loglevel) + + # logging other modules + for logger in ("message", "database", "storage", "tsdb"): + logger_config = self.main_config.to_dict()[logger] + logger_module = logging.getLogger(logger_config["logger_name"]) + if logger_config["logfile"]: + file_handler = logging.handlers.RotatingFileHandler( + logger_config["logfile"], maxBytes=100e6, backupCount=9, delay=0 + ) + file_handler.setFormatter(log_formatter_simple) + logger_module.addHandler(file_handler) + if logger_config["loglevel"]: + logger_module.setLevel(logger_config["loglevel"]) + self.logger.critical("starting osm/nglcm") + + try: + self.db = Database(self.main_config.to_dict()).instance.db - except (DbException) as e: ++ except DbException as e: + self.logger.critical(str(e), exc_info=True) + raise LcmException(str(e)) + + async def start(self): + # do some temporal stuff here + temporal_api = ( + f"{self.main_config.temporal.host}:{str(self.main_config.temporal.port)}" + ) + self.logger.info(f"Attempting to register with Temporal at {temporal_api}") + client = await Client.connect(temporal_api) + + task_queue = "lcm-task-queue" + workflows = [ + Heartbeat, + ] + activities = [] + + worker = Worker( + client, task_queue=task_queue, workflows=workflows, activities=activities + ) + + self.logger.info(f"Registered for queue {task_queue}") + self.logger.info(f"Registered workflows {workflows}") + self.logger.info(f"Registered activites {activities}") + + await worker.run() + + def read_config_file(self, config_file): + try: + with open(config_file) as f: + return yaml.safe_load(f) + except Exception as e: + self.logger.critical("At config file '{}': {}".format(config_file, e)) + exit(1) + + +@workflow.defn +class Heartbeat: + @workflow.run + async def run(self) -> str: + return "alive" + + +if __name__ == "__main__": - + try: + opts, args = getopt.getopt( + sys.argv[1:], "hc:", ["config=", "help", "health-check"] + ) + # TODO add "log-socket-host=", "log-socket-port=", "log-file=" + config_file = None + for o, a in opts: + if o in ("-c", "--config"): + config_file = a + else: + assert False, "Unhandled option" + + if config_file: + if not path.isfile(config_file): + print( + "configuration file '{}' does not exist".format(config_file), + file=sys.stderr, + ) + exit(1) + else: + for config_file in ( + __file__[: __file__.rfind(".")] + ".cfg", + "./lcm.cfg", + "/etc/osm/lcm.cfg", + ): + print(f"{config_file}") + if path.isfile(config_file): + break + else: + print( + "No configuration file 'lcm.cfg' found neither at local folder nor at /etc/osm/", + file=sys.stderr, + ) + exit(1) + lcm = NGLcm(config_file) + asyncio.run(lcm.start()) + except (LcmException, getopt.GetoptError) as e: + print(str(e), file=sys.stderr) + # usage() + exit(1) diff --cc requirements-dev.txt index 2ec95c2,b9bb15d..2c85a0f --- a/requirements-dev.txt +++ b/requirements-dev.txt @@@ -14,23 -14,25 +14,25 @@@ # See the License for the specific language governing permissions and # limitations under the License. ####################################################################################### - aiokafka==0.7.2 + aiokafka==0.8.0 - # via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master + # via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=paas - async-timeout==3.0.1 + async-timeout==4.0.2 # via - # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=master - # -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master + # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=paas ++ # -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=paas + # aiokafka # retrying-async bcrypt==4.0.1 # via - # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=master + # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=paas # paramiko - cachetools==5.2.0 + cachetools==5.3.0 # via - # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=master + # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=paas # google-auth - certifi==2022.9.24 + certifi==2022.12.7 # via - # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=master + # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=paas # kubernetes # requests cffi==1.15.1 @@@ -40,17 -42,17 +42,17 @@@ # pynacl charset-normalizer==2.1.1 # via - # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=master + # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=paas # requests - cryptography==38.0.1 + cryptography==39.0.0 # via - # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=master + # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=paas # paramiko dataclasses==0.6 - # via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master + # via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=paas - google-auth==2.12.0 + google-auth==2.16.0 # via - # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=master + # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=paas # kubernetes idna==3.4 # via @@@ -64,11 -66,11 +66,11 @@@ jujubundlelib==0.5. # theblues kafka-python==2.0.2 # via - # -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master + # -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=paas # aiokafka - kubernetes==24.2.0 + kubernetes==25.3.0 # via - # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=master + # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=paas # juju macaroonbakery==1.3.1 # via @@@ -76,22 -78,28 +78,28 @@@ # juju # theblues motor==1.3.1 - # via -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=paas + # via - # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=master - # -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master ++ # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=paas ++ # -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=paas mypy-extensions==0.4.3 # via - # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=master + # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=paas # typing-inspect -n2vc @ git+https://osm.etsi.org/gerrit/osm/N2VC.git@master +n2vc @ git+https://osm.etsi.org/gerrit/osm/N2VC.git@paas # via -r requirements-dev.in - oauthlib==3.2.1 + oauthlib==3.2.2 # via - # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=master + # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=paas # requests-oauthlib -osm-common @ git+https://osm.etsi.org/gerrit/osm/common.git@master +osm-common @ git+https://osm.etsi.org/gerrit/osm/common.git@paas # via -r requirements-dev.in - paramiko==2.11.0 + packaging==23.0 + # via - # -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master ++ # -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=paas + # aiokafka + paramiko==2.12.0 # via - # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=master + # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=paas # juju protobuf==3.20.3 # via @@@ -109,18 -117,18 +117,18 @@@ pyasn1-modules==0.2. # google-auth pycparser==2.21 # via - # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=master + # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=paas # cffi - pycrypto==2.6.1 + pycryptodome==3.17 - # via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master + # via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=paas pymacaroons==0.13.0 # via - # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=master + # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=paas # macaroonbakery - pymongo==3.12.3 + pymongo==3.13.0 # via - # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=master - # -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master + # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=paas + # -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=paas # motor pynacl==1.5.0 # via @@@ -135,11 -143,11 +143,11 @@@ pyrfc3339==1. # macaroonbakery python-dateutil==2.8.2 # via - # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=master + # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=paas # kubernetes - pytz==2022.4 + pytz==2022.7.1 # via - # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=master + # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=paas # pyrfc3339 pyyaml==5.4.1 # via @@@ -148,9 -156,9 +156,9 @@@ # juju # jujubundlelib # kubernetes - requests==2.28.1 + requests==2.28.2 # via - # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=master + # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=paas # kubernetes # macaroonbakery # requests-oauthlib @@@ -176,11 -184,11 +184,11 @@@ six==1.16. # python-dateutil theblues==0.5.2 # via - # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=master + # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=paas # juju - toposort==1.7 + toposort==1.9 # via - # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=master + # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=paas # juju typing-extensions==4.4.0 # via @@@ -188,16 -196,16 +196,16 @@@ # typing-inspect typing-inspect==0.8.0 # via - # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=master + # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=paas # juju - urllib3==1.26.12 + urllib3==1.26.14 # via - # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=master + # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=paas # kubernetes # requests - websocket-client==1.4.1 + websocket-client==1.5.0 # via - # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=master + # -r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=paas # kubernetes websockets==7.0 # via diff --cc requirements.in index 782bef5,67d3cfe..8a96125 --- a/requirements.in +++ b/requirements.in @@@ -22,5 -23,3 +23,4 @@@ jinja pyyaml==5.4.1 pydantic protobuf==3.20.3 - config-man==0.0.4 - temporalio ++temporalio diff --cc requirements.txt index a14a076,4ea9719..5d912be --- a/requirements.txt +++ b/requirements.txt @@@ -67,24 -73,12 +73,23 @@@ protobuf==3.20. # via # -r requirements.in # grpcio-tools + # temporalio pydantic==1.10.4 # via -r requirements.in +python-dateutil==2.8.2 + # via temporalio pyyaml==5.4.1 # via -r requirements.in +six==1.16.0 + # via python-dateutil +temporalio==1.0.0 + # via -r requirements.in +types-protobuf==3.20.4.6 + # via temporalio typing-extensions==4.4.0 - # via pydantic + # via - # aiohttp + # pydantic + # temporalio yarl==1.8.2 # via aiohttp