From 821bfc9e2aa6631269a6f2d4da4961c46586e28e Mon Sep 17 00:00:00 2001 From: Mark Beierl Date: Tue, 24 Jan 2023 21:15:25 -0500 Subject: [PATCH] Initial Temporal Config Adds a new main (nglcm.py) and config file for handling Temporal connectivity Change-Id: I0d30897b65fa4e9541d341f331983696ac5536ba Signed-off-by: Mark Beierl --- osm_lcm/data_utils/lcm_config.py | 15 +++ osm_lcm/nglcm.cfg | 62 +++++++++++ osm_lcm/nglcm.py | 185 +++++++++++++++++++++++++++++++ requirements-dev.in | 8 +- requirements-test.txt | 4 +- requirements.in | 3 +- requirements.txt | 26 +++-- 7 files changed, 287 insertions(+), 16 deletions(-) create mode 100644 osm_lcm/nglcm.cfg create mode 100644 osm_lcm/nglcm.py diff --git a/osm_lcm/data_utils/lcm_config.py b/osm_lcm/data_utils/lcm_config.py index 08a8728..ffd236e 100644 --- a/osm_lcm/data_utils/lcm_config.py +++ b/osm_lcm/data_utils/lcm_config.py @@ -198,6 +198,20 @@ class TsdbConfig(OsmConfigman): self.logger_name = "lcm.prometheus" +class TemporalConfig(OsmConfigman): + driver: str = None + path: str = None + host: str = None + port: int = None + loglevel: str = "DEBUG" + logfile: str = None + group_id: str = None + logger_name: str = None + + def transform(self): + self.logger_name = "lcm.temporal" + + # Main configuration Template @@ -210,6 +224,7 @@ class LcmCfg(OsmConfigman): storage: StorageConfig = StorageConfig() message: MessageConfig = MessageConfig() tsdb: TsdbConfig = TsdbConfig() + temporal: TemporalConfig = TemporalConfig() def transform(self): for attribute in dir(self): diff --git a/osm_lcm/nglcm.cfg b/osm_lcm/nglcm.cfg new file mode 100644 index 0000000..be9658b --- /dev/null +++ b/osm_lcm/nglcm.cfg @@ -0,0 +1,62 @@ +## +# Copyright 2018 Telefonica S.A. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## + +# TODO currently is a pure yaml format. Consider to change it to [ini: style with yaml inside to be coherent with other modules + +global: + loglevel: DEBUG + # logfile: /app/log # or /var/log/osm/lcm.log + # nologging: True # do no log to stdout/stderr + +VCA: + host: vca + port: 17070 + user: admin + secret: secret + cloud: localhost + k8s_cloud: k8scloud + helmpath: /usr/local/bin/helm + helm3path: /usr/local/bin/helm3 + kubectlpath: /usr/bin/kubectl + jujupath: /usr/local/bin/juju + eegrpc_tls_enforce: false + # public_key: pubkey + # ca_cert: cacert + # api_proxy: apiproxy + # eegrpcinittimeout: 600 + # eegrpctimeout: 30 + + # loglevel: DEBUG + # logfile: /var/log/osm/lcm-vca.log + +database: + driver: mongo # mongo or memory + host: mongo # hostname or IP + port: 27017 + name: osm + # replicaset: replicaset + # user: user + # password: password + # commonkey: "XXXXXX" # password used for encryption of sensible information + # loglevel: DEBUG + # logfile: /var/log/osm/lcm-database.log + +storage: + driver: local # local filesystem + # for local provide file path + path: /app/storage + # loglevel: DEBUG + # logfile: /var/log/osm/lcm-storage.log diff --git a/osm_lcm/nglcm.py b/osm_lcm/nglcm.py new file mode 100644 index 0000000..67b2a3f --- /dev/null +++ b/osm_lcm/nglcm.py @@ -0,0 +1,185 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +## +# Copyright 2018 Telefonica S.A. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## + + +import asyncio +import getopt +import logging +import logging.handlers +import sys +import yaml + +from osm_lcm.lcm_utils import LcmException + +from osm_common.dbbase import DbException +from osm_lcm.data_utils.database.database import Database +from osm_lcm.data_utils.lcm_config import LcmCfg +from os import path +from temporalio import workflow +from temporalio.client import Client +from temporalio.worker import Worker + + +class NGLcm: + + main_config = LcmCfg() + + def __init__(self, config_file, loop=None): + """ + Init, Connect to database, filesystem storage, and messaging + :param config: two level dictionary with configuration. Top level should contain 'database', 'storage', + :return: None + """ + self.db = None + + # logging + self.logger = logging.getLogger("lcm") + # load configuration + config = self.read_config_file(config_file) + self.main_config.set_from_dict(config) + self.main_config.transform() + self.main_config.load_from_env() + self.logger.critical("Loaded configuration:" + str(self.main_config.to_dict())) + + # logging + log_format_simple = ( + "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s" + ) + log_formatter_simple = logging.Formatter( + log_format_simple, datefmt="%Y-%m-%dT%H:%M:%S" + ) + if self.main_config.globalConfig.logfile: + file_handler = logging.handlers.RotatingFileHandler( + self.main_config.globalConfig.logfile, + maxBytes=100e6, + backupCount=9, + delay=0, + ) + file_handler.setFormatter(log_formatter_simple) + self.logger.addHandler(file_handler) + if not self.main_config.globalConfig.to_dict()["nologging"]: + str_handler = logging.StreamHandler() + str_handler.setFormatter(log_formatter_simple) + self.logger.addHandler(str_handler) + + if self.main_config.globalConfig.to_dict()["loglevel"]: + self.logger.setLevel(self.main_config.globalConfig.loglevel) + + # logging other modules + for logger in ("message", "database", "storage", "tsdb"): + logger_config = self.main_config.to_dict()[logger] + logger_module = logging.getLogger(logger_config["logger_name"]) + if logger_config["logfile"]: + file_handler = logging.handlers.RotatingFileHandler( + logger_config["logfile"], maxBytes=100e6, backupCount=9, delay=0 + ) + file_handler.setFormatter(log_formatter_simple) + logger_module.addHandler(file_handler) + if logger_config["loglevel"]: + logger_module.setLevel(logger_config["loglevel"]) + self.logger.critical("starting osm/nglcm") + + try: + self.db = Database(self.main_config.to_dict()).instance.db + except (DbException) as e: + self.logger.critical(str(e), exc_info=True) + raise LcmException(str(e)) + + async def start(self): + # do some temporal stuff here + temporal_api = ( + f"{self.main_config.temporal.host}:{str(self.main_config.temporal.port)}" + ) + self.logger.info(f"Attempting to register with Temporal at {temporal_api}") + client = await Client.connect(temporal_api) + + task_queue = "lcm-task-queue" + workflows = [ + Heartbeat, + ] + activities = [] + + worker = Worker( + client, task_queue=task_queue, workflows=workflows, activities=activities + ) + + self.logger.info(f"Registered for queue {task_queue}") + self.logger.info(f"Registered workflows {workflows}") + self.logger.info(f"Registered activites {activities}") + + await worker.run() + + def read_config_file(self, config_file): + try: + with open(config_file) as f: + return yaml.safe_load(f) + except Exception as e: + self.logger.critical("At config file '{}': {}".format(config_file, e)) + exit(1) + + +@workflow.defn +class Heartbeat: + @workflow.run + async def run(self) -> str: + return "alive" + + +if __name__ == "__main__": + + try: + opts, args = getopt.getopt( + sys.argv[1:], "hc:", ["config=", "help", "health-check"] + ) + # TODO add "log-socket-host=", "log-socket-port=", "log-file=" + config_file = None + for o, a in opts: + if o in ("-c", "--config"): + config_file = a + else: + assert False, "Unhandled option" + + if config_file: + if not path.isfile(config_file): + print( + "configuration file '{}' does not exist".format(config_file), + file=sys.stderr, + ) + exit(1) + else: + for config_file in ( + __file__[: __file__.rfind(".")] + ".cfg", + "./lcm.cfg", + "/etc/osm/lcm.cfg", + ): + print(f"{config_file}") + if path.isfile(config_file): + break + else: + print( + "No configuration file 'lcm.cfg' found neither at local folder nor at /etc/osm/", + file=sys.stderr, + ) + exit(1) + lcm = NGLcm(config_file) + asyncio.run(lcm.start()) + except (LcmException, getopt.GetoptError) as e: + print(str(e), file=sys.stderr) + # usage() + exit(1) diff --git a/requirements-dev.in b/requirements-dev.in index ec714a2..cfa3e71 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -git+https://osm.etsi.org/gerrit/osm/common.git@master#egg=osm-common --r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master +git+https://osm.etsi.org/gerrit/osm/common.git@paas#egg=osm-common +-r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=paas -git+https://osm.etsi.org/gerrit/osm/N2VC.git@master#egg=n2vc --r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=master +git+https://osm.etsi.org/gerrit/osm/N2VC.git@paas#egg=n2vc +-r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=paas diff --git a/requirements-test.txt b/requirements-test.txt index fb86ef4..d157b15 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -16,9 +16,9 @@ ####################################################################################### asynctest==0.13.0 # via -r requirements-test.in -coverage==6.5.0 +coverage==7.0.5 # via -r requirements-test.in -mock==4.0.3 +mock==5.0.1 # via -r requirements-test.in nose2==0.12.0 # via -r requirements-test.in diff --git a/requirements.in b/requirements.in index 41a01ba..782bef5 100644 --- a/requirements.in +++ b/requirements.in @@ -22,4 +22,5 @@ jinja2 pyyaml==5.4.1 pydantic protobuf==3.20.3 -config-man==0.0.4 \ No newline at end of file +config-man==0.0.4 +temporalio \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 24c09c0..a14a076 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ async-timeout==3.0.1 # via # -r requirements.in # aiohttp -attrs==22.1.0 +attrs==22.2.0 # via # aiohttp # glom @@ -34,11 +34,11 @@ checksumdir==1.2.0 # via -r requirements.in config-man==0.0.4 # via -r requirements.in -face==22.0.0 +face==20.1.1 # via glom -glom==22.1.0 +glom==23.1.1 # via config-man -grpcio==1.50.0 +grpcio==1.51.1 # via grpcio-tools grpcio-tools==1.48.1 # via -r requirements.in @@ -56,9 +56,9 @@ idna==3.4 # yarl jinja2==3.1.2 # via -r requirements.in -markupsafe==2.1.1 +markupsafe==2.1.2 # via jinja2 -multidict==6.0.2 +multidict==6.0.4 # via # aiohttp # grpclib @@ -67,17 +67,25 @@ protobuf==3.20.3 # via # -r requirements.in # grpcio-tools -pydantic==1.10.2 + # temporalio +pydantic==1.10.4 # via -r requirements.in +python-dateutil==2.8.2 + # via temporalio pyyaml==5.4.1 # via -r requirements.in six==1.16.0 - # via grpcio + # via python-dateutil +temporalio==1.0.0 + # via -r requirements.in +types-protobuf==3.20.4.6 + # via temporalio typing-extensions==4.4.0 # via # aiohttp # pydantic -yarl==1.8.1 + # temporalio +yarl==1.8.2 # via aiohttp # The following packages are considered to be unsafe in a requirements file: -- 2.25.1