Initial Temporal Config 60/12860/3
authorMark Beierl <mark.beierl@canonical.com>
Wed, 25 Jan 2023 02:15:25 +0000 (21:15 -0500)
committerMark Beierl <mark.beierl@canonical.com>
Wed, 25 Jan 2023 16:35:04 +0000 (11:35 -0500)
Adds a new main (nglcm.py) and config file for handling Temporal
connectivity

Change-Id: I0d30897b65fa4e9541d341f331983696ac5536ba
Signed-off-by: Mark Beierl <mark.beierl@canonical.com>
osm_lcm/data_utils/lcm_config.py
osm_lcm/nglcm.cfg [new file with mode: 0644]
osm_lcm/nglcm.py [new file with mode: 0644]
requirements-dev.in
requirements-test.txt
requirements.in
requirements.txt

index 08a8728..ffd236e 100644 (file)
@@ -198,6 +198,20 @@ class TsdbConfig(OsmConfigman):
         self.logger_name = "lcm.prometheus"
 
 
+class TemporalConfig(OsmConfigman):
+    driver: str = None
+    path: str = None
+    host: str = None
+    port: int = None
+    loglevel: str = "DEBUG"
+    logfile: str = None
+    group_id: str = None
+    logger_name: str = None
+
+    def transform(self):
+        self.logger_name = "lcm.temporal"
+
+
 # Main configuration Template
 
 
@@ -210,6 +224,7 @@ class LcmCfg(OsmConfigman):
     storage: StorageConfig = StorageConfig()
     message: MessageConfig = MessageConfig()
     tsdb: TsdbConfig = TsdbConfig()
+    temporal: TemporalConfig = TemporalConfig()
 
     def transform(self):
         for attribute in dir(self):
diff --git a/osm_lcm/nglcm.cfg b/osm_lcm/nglcm.cfg
new file mode 100644 (file)
index 0000000..be9658b
--- /dev/null
@@ -0,0 +1,62 @@
+##
+# Copyright 2018 Telefonica S.A.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+##
+
+# TODO currently is a pure yaml format. Consider to change it to [ini: style with yaml inside to be coherent with other modules
+
+global:
+    loglevel: DEBUG
+    # logfile:  /app/log  # or /var/log/osm/lcm.log
+    # nologging: True     # do no log to stdout/stderr
+
+VCA:
+    host:   vca
+    port:   17070
+    user:   admin
+    secret: secret
+    cloud:  localhost
+    k8s_cloud: k8scloud
+    helmpath:    /usr/local/bin/helm
+    helm3path:    /usr/local/bin/helm3
+    kubectlpath: /usr/bin/kubectl
+    jujupath:    /usr/local/bin/juju
+    eegrpc_tls_enforce: false
+    # public_key: pubkey
+    # ca_cert: cacert
+    # api_proxy: apiproxy
+    # eegrpcinittimeout: 600
+    # eegrpctimeout: 30
+
+    # loglevel: DEBUG
+    # logfile:  /var/log/osm/lcm-vca.log
+
+database:
+    driver: mongo       # mongo or memory
+    host:   mongo       # hostname or IP
+    port:   27017
+    name:   osm
+    # replicaset: replicaset
+    # user:   user
+    # password:   password
+    # commonkey: "XXXXXX" # password used for encryption of sensible information
+    # loglevel: DEBUG
+    # logfile:  /var/log/osm/lcm-database.log
+
+storage:
+    driver: local       # local filesystem
+    # for local provide file path
+    path:   /app/storage
+    # loglevel: DEBUG
+    # logfile:  /var/log/osm/lcm-storage.log
diff --git a/osm_lcm/nglcm.py b/osm_lcm/nglcm.py
new file mode 100644 (file)
index 0000000..67b2a3f
--- /dev/null
@@ -0,0 +1,185 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+##
+# Copyright 2018 Telefonica S.A.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+##
+
+
+import asyncio
+import getopt
+import logging
+import logging.handlers
+import sys
+import yaml
+
+from osm_lcm.lcm_utils import LcmException
+
+from osm_common.dbbase import DbException
+from osm_lcm.data_utils.database.database import Database
+from osm_lcm.data_utils.lcm_config import LcmCfg
+from os import path
+from temporalio import workflow
+from temporalio.client import Client
+from temporalio.worker import Worker
+
+
+class NGLcm:
+
+    main_config = LcmCfg()
+
+    def __init__(self, config_file, loop=None):
+        """
+        Init, Connect to database, filesystem storage, and messaging
+        :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
+        :return: None
+        """
+        self.db = None
+
+        # logging
+        self.logger = logging.getLogger("lcm")
+        # load configuration
+        config = self.read_config_file(config_file)
+        self.main_config.set_from_dict(config)
+        self.main_config.transform()
+        self.main_config.load_from_env()
+        self.logger.critical("Loaded configuration:" + str(self.main_config.to_dict()))
+
+        # logging
+        log_format_simple = (
+            "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
+        )
+        log_formatter_simple = logging.Formatter(
+            log_format_simple, datefmt="%Y-%m-%dT%H:%M:%S"
+        )
+        if self.main_config.globalConfig.logfile:
+            file_handler = logging.handlers.RotatingFileHandler(
+                self.main_config.globalConfig.logfile,
+                maxBytes=100e6,
+                backupCount=9,
+                delay=0,
+            )
+            file_handler.setFormatter(log_formatter_simple)
+            self.logger.addHandler(file_handler)
+        if not self.main_config.globalConfig.to_dict()["nologging"]:
+            str_handler = logging.StreamHandler()
+            str_handler.setFormatter(log_formatter_simple)
+            self.logger.addHandler(str_handler)
+
+        if self.main_config.globalConfig.to_dict()["loglevel"]:
+            self.logger.setLevel(self.main_config.globalConfig.loglevel)
+
+        # logging other modules
+        for logger in ("message", "database", "storage", "tsdb"):
+            logger_config = self.main_config.to_dict()[logger]
+            logger_module = logging.getLogger(logger_config["logger_name"])
+            if logger_config["logfile"]:
+                file_handler = logging.handlers.RotatingFileHandler(
+                    logger_config["logfile"], maxBytes=100e6, backupCount=9, delay=0
+                )
+                file_handler.setFormatter(log_formatter_simple)
+                logger_module.addHandler(file_handler)
+            if logger_config["loglevel"]:
+                logger_module.setLevel(logger_config["loglevel"])
+        self.logger.critical("starting osm/nglcm")
+
+        try:
+            self.db = Database(self.main_config.to_dict()).instance.db
+        except (DbException) as e:
+            self.logger.critical(str(e), exc_info=True)
+            raise LcmException(str(e))
+
+    async def start(self):
+        # do some temporal stuff here
+        temporal_api = (
+            f"{self.main_config.temporal.host}:{str(self.main_config.temporal.port)}"
+        )
+        self.logger.info(f"Attempting to register with Temporal at {temporal_api}")
+        client = await Client.connect(temporal_api)
+
+        task_queue = "lcm-task-queue"
+        workflows = [
+            Heartbeat,
+        ]
+        activities = []
+
+        worker = Worker(
+            client, task_queue=task_queue, workflows=workflows, activities=activities
+        )
+
+        self.logger.info(f"Registered for queue {task_queue}")
+        self.logger.info(f"Registered workflows {workflows}")
+        self.logger.info(f"Registered activites {activities}")
+
+        await worker.run()
+
+    def read_config_file(self, config_file):
+        try:
+            with open(config_file) as f:
+                return yaml.safe_load(f)
+        except Exception as e:
+            self.logger.critical("At config file '{}': {}".format(config_file, e))
+            exit(1)
+
+
+@workflow.defn
+class Heartbeat:
+    @workflow.run
+    async def run(self) -> str:
+        return "alive"
+
+
+if __name__ == "__main__":
+
+    try:
+        opts, args = getopt.getopt(
+            sys.argv[1:], "hc:", ["config=", "help", "health-check"]
+        )
+        # TODO add  "log-socket-host=", "log-socket-port=", "log-file="
+        config_file = None
+        for o, a in opts:
+            if o in ("-c", "--config"):
+                config_file = a
+            else:
+                assert False, "Unhandled option"
+
+        if config_file:
+            if not path.isfile(config_file):
+                print(
+                    "configuration file '{}' does not exist".format(config_file),
+                    file=sys.stderr,
+                )
+                exit(1)
+        else:
+            for config_file in (
+                __file__[: __file__.rfind(".")] + ".cfg",
+                "./lcm.cfg",
+                "/etc/osm/lcm.cfg",
+            ):
+                print(f"{config_file}")
+                if path.isfile(config_file):
+                    break
+            else:
+                print(
+                    "No configuration file 'lcm.cfg' found neither at local folder nor at /etc/osm/",
+                    file=sys.stderr,
+                )
+                exit(1)
+        lcm = NGLcm(config_file)
+        asyncio.run(lcm.start())
+    except (LcmException, getopt.GetoptError) as e:
+        print(str(e), file=sys.stderr)
+        # usage()
+        exit(1)
index ec714a2..cfa3e71 100644 (file)
@@ -13,8 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-git+https://osm.etsi.org/gerrit/osm/common.git@master#egg=osm-common
--r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master
+git+https://osm.etsi.org/gerrit/osm/common.git@paas#egg=osm-common
+-r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=paas
 
-git+https://osm.etsi.org/gerrit/osm/N2VC.git@master#egg=n2vc
--r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=master
+git+https://osm.etsi.org/gerrit/osm/N2VC.git@paas#egg=n2vc
+-r https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=blob_plain;f=requirements.txt;hb=paas
index fb86ef4..d157b15 100644 (file)
@@ -16,9 +16,9 @@
 #######################################################################################
 asynctest==0.13.0
     # via -r requirements-test.in
-coverage==6.5.0
+coverage==7.0.5
     # via -r requirements-test.in
-mock==4.0.3
+mock==5.0.1
     # via -r requirements-test.in
 nose2==0.12.0
     # via -r requirements-test.in
index 41a01ba..782bef5 100644 (file)
@@ -22,4 +22,5 @@ jinja2
 pyyaml==5.4.1
 pydantic
 protobuf==3.20.3
-config-man==0.0.4
\ No newline at end of file
+config-man==0.0.4
+temporalio
\ No newline at end of file
index 24c09c0..a14a076 100644 (file)
@@ -20,7 +20,7 @@ async-timeout==3.0.1
     # via
     #   -r requirements.in
     #   aiohttp
-attrs==22.1.0
+attrs==22.2.0
     # via
     #   aiohttp
     #   glom
@@ -34,11 +34,11 @@ checksumdir==1.2.0
     # via -r requirements.in
 config-man==0.0.4
     # via -r requirements.in
-face==22.0.0
+face==20.1.1
     # via glom
-glom==22.1.0
+glom==23.1.1
     # via config-man
-grpcio==1.50.0
+grpcio==1.51.1
     # via grpcio-tools
 grpcio-tools==1.48.1
     # via -r requirements.in
@@ -56,9 +56,9 @@ idna==3.4
     #   yarl
 jinja2==3.1.2
     # via -r requirements.in
-markupsafe==2.1.1
+markupsafe==2.1.2
     # via jinja2
-multidict==6.0.2
+multidict==6.0.4
     # via
     #   aiohttp
     #   grpclib
@@ -67,17 +67,25 @@ protobuf==3.20.3
     # via
     #   -r requirements.in
     #   grpcio-tools
-pydantic==1.10.2
+    #   temporalio
+pydantic==1.10.4
     # via -r requirements.in
+python-dateutil==2.8.2
+    # via temporalio
 pyyaml==5.4.1
     # via -r requirements.in
 six==1.16.0
-    # via grpcio
+    # via python-dateutil
+temporalio==1.0.0
+    # via -r requirements.in
+types-protobuf==3.20.4.6
+    # via temporalio
 typing-extensions==4.4.0
     # via
     #   aiohttp
     #   pydantic
-yarl==1.8.1
+    #   temporalio
+yarl==1.8.2
     # via aiohttp
 
 # The following packages are considered to be unsafe in a requirements file: