Coverage for NG-RO/osm_ng_ro/ro_main.py: 0%
459 statements
« prev ^ index » next coverage.py v7.6.12, created at 2025-04-14 12:04 +0000
« prev ^ index » next coverage.py v7.6.12, created at 2025-04-14 12:04 +0000
1#!/usr/bin/python3
2# -*- coding: utf-8 -*-
4##
5# Copyright 2020 Telefonica Investigacion y Desarrollo, S.A.U.
6#
7# Licensed under the Apache License, Version 2.0 (the "License");
8# you may not use this file except in compliance with the License.
9# You may obtain a copy of the License at
10#
11# http://www.apache.org/licenses/LICENSE-2.0
12#
13# Unless required by applicable law or agreed to in writing, software
14# distributed under the License is distributed on an "AS IS" BASIS,
15# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
16# implied.
17# See the License for the specific language governing permissions and
18# limitations under the License.
19##
22from codecs import getreader
23import getopt
24from http import HTTPStatus
25import json
26import logging
27import logging.handlers
28from os import environ, path
29import sys
30import time
32import cherrypy
33from osm_common.dbbase import DbException
34from osm_common.fsbase import FsException
35from osm_common.msgbase import MsgException
36from osm_ng_ro import version as ro_version, version_date as ro_version_date
37import osm_ng_ro.html_out as html
38from osm_ng_ro.monitor import start_monitoring, stop_monitoring
39from osm_ng_ro.ns import Ns, NsException
40from osm_ng_ro.validation import ValidationError
41from osm_ng_ro.vim_admin import VimAdminThread
42import yaml
45__author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
46__version__ = "0.1." # file version, not NBI version
47version_date = "May 2020"
49database_version = "1.2"
50auth_database_version = "1.0"
51ro_server = None # instance of Server class
52vim_admin_thread = None # instance of VimAdminThread class
54# vim_threads = None # instance of VimThread class
56"""
57RO North Bound Interface
58URL: /ro GET POST PUT DELETE PATCH
59 /ns/v1/deploy O
60 /<nsrs_id> O O O
61 /<action_id> O
62 /cancel O
64"""
66valid_query_string = ("ADMIN", "SET_PROJECT", "FORCE", "PUBLIC")
67# ^ Contains possible administrative query string words:
68# ADMIN=True(by default)|Project|Project-list: See all elements, or elements of a project
69# (not owned by my session project).
70# PUBLIC=True(by default)|False: See/hide public elements. Set/Unset a topic to be public
71# FORCE=True(by default)|False: Force edition/deletion operations
72# SET_PROJECT=Project|Project-list: Add/Delete the topic to the projects portfolio
74valid_url_methods = {
75 # contains allowed URL and methods, and the role_permission name
76 "admin": {
77 "v1": {
78 "tokens": {
79 "METHODS": ("POST",),
80 "ROLE_PERMISSION": "tokens:",
81 "<ID>": {"METHODS": ("DELETE",), "ROLE_PERMISSION": "tokens:id:"},
82 },
83 }
84 },
85 "ns": {
86 "v1": {
87 "rebuild": {
88 "METHODS": ("POST",),
89 "ROLE_PERMISSION": "rebuild:",
90 "<ID>": {
91 "METHODS": ("POST",),
92 "ROLE_PERMISSION": "rebuild:id:",
93 },
94 },
95 "start": {
96 "METHODS": ("POST",),
97 "ROLE_PERMISSION": "start:",
98 "<ID>": {
99 "METHODS": ("POST",),
100 "ROLE_PERMISSION": "start:id:",
101 },
102 },
103 "stop": {
104 "METHODS": ("POST",),
105 "ROLE_PERMISSION": "stop:",
106 "<ID>": {
107 "METHODS": ("POST",),
108 "ROLE_PERMISSION": "stop:id:",
109 },
110 },
111 "deploy": {
112 "METHODS": ("GET",),
113 "ROLE_PERMISSION": "deploy:",
114 "<ID>": {
115 "METHODS": ("GET", "POST", "DELETE"),
116 "ROLE_PERMISSION": "deploy:id:",
117 "<ID>": {
118 "METHODS": ("GET",),
119 "ROLE_PERMISSION": "deploy:id:id:",
120 "cancel": {
121 "METHODS": ("POST",),
122 "ROLE_PERMISSION": "deploy:id:id:cancel",
123 },
124 },
125 },
126 },
127 "recreate": {
128 "<ID>": {
129 "METHODS": ("POST"),
130 "ROLE_PERMISSION": "recreate:id:",
131 "<ID>": {
132 "METHODS": ("GET",),
133 "ROLE_PERMISSION": "recreate:id:id:",
134 },
135 },
136 },
137 "migrate": {
138 "<ID>": {
139 "METHODS": ("POST"),
140 "ROLE_PERMISSION": "migrate:id:",
141 "<ID>": {
142 "METHODS": ("GET",),
143 "ROLE_PERMISSION": "migrate:id:id:",
144 },
145 },
146 },
147 "verticalscale": {
148 "<ID>": {
149 "METHODS": ("POST"),
150 "ROLE_PERMISSION": "verticalscale:id:",
151 "<ID>": {
152 "METHODS": ("GET",),
153 "ROLE_PERMISSION": "verticalscale:id:id:",
154 },
155 },
156 },
157 }
158 },
159}
162class RoException(Exception):
163 def __init__(self, message, http_code=HTTPStatus.METHOD_NOT_ALLOWED):
164 Exception.__init__(self, message)
165 self.http_code = http_code
168class AuthException(RoException):
169 pass
172class Authenticator:
173 def __init__(self, valid_url_methods, valid_query_string):
174 self.valid_url_methods = valid_url_methods
175 self.valid_query_string = valid_query_string
177 def authorize(self, *args, **kwargs):
178 return {"token": "ok", "id": "ok"}
180 def new_token(self, token_info, indata, remote):
181 return {"token": "ok", "id": "ok", "remote": remote}
183 def del_token(self, token_id):
184 pass
186 def start(self, engine_config):
187 pass
190class Server(object):
191 instance = 0
192 # to decode bytes to str
193 reader = getreader("utf-8")
195 def __init__(self):
196 self.instance += 1
197 self.authenticator = Authenticator(valid_url_methods, valid_query_string)
198 self.ns = Ns()
199 self.map_operation = {
200 "token:post": self.new_token,
201 "token:id:delete": self.del_token,
202 "deploy:get": self.ns.get_deploy,
203 "deploy:id:get": self.ns.get_actions,
204 "deploy:id:post": self.ns.deploy,
205 "deploy:id:delete": self.ns.delete,
206 "deploy:id:id:get": self.ns.status,
207 "deploy:id:id:cancel:post": self.ns.cancel,
208 "rebuild:id:post": self.ns.rebuild_start_stop,
209 "start:id:post": self.ns.rebuild_start_stop,
210 "stop:id:post": self.ns.rebuild_start_stop,
211 "recreate:id:post": self.ns.recreate,
212 "recreate:id:id:get": self.ns.recreate_status,
213 "migrate:id:post": self.ns.migrate,
214 "verticalscale:id:post": self.ns.verticalscale,
215 }
217 def _format_in(self, kwargs):
218 error_text = ""
219 try:
220 indata = None
222 if cherrypy.request.body.length:
223 error_text = "Invalid input format "
225 if "Content-Type" in cherrypy.request.headers:
226 if "application/json" in cherrypy.request.headers["Content-Type"]:
227 error_text = "Invalid json format "
228 indata = json.load(self.reader(cherrypy.request.body))
229 cherrypy.request.headers.pop("Content-File-MD5", None)
230 elif "application/yaml" in cherrypy.request.headers["Content-Type"]:
231 error_text = "Invalid yaml format "
232 indata = yaml.safe_load(cherrypy.request.body)
233 cherrypy.request.headers.pop("Content-File-MD5", None)
234 elif (
235 "application/binary" in cherrypy.request.headers["Content-Type"]
236 or "application/gzip"
237 in cherrypy.request.headers["Content-Type"]
238 or "application/zip" in cherrypy.request.headers["Content-Type"]
239 or "text/plain" in cherrypy.request.headers["Content-Type"]
240 ):
241 indata = cherrypy.request.body # .read()
242 elif (
243 "multipart/form-data"
244 in cherrypy.request.headers["Content-Type"]
245 ):
246 if "descriptor_file" in kwargs:
247 filecontent = kwargs.pop("descriptor_file")
249 if not filecontent.file:
250 raise RoException(
251 "empty file or content", HTTPStatus.BAD_REQUEST
252 )
254 indata = filecontent.file # .read()
256 if filecontent.content_type.value:
257 cherrypy.request.headers["Content-Type"] = (
258 filecontent.content_type.value
259 )
260 else:
261 # raise cherrypy.HTTPError(HTTPStatus.Not_Acceptable,
262 # "Only 'Content-Type' of type 'application/json' or
263 # 'application/yaml' for input format are available")
264 error_text = "Invalid yaml format "
265 indata = yaml.safe_load(cherrypy.request.body)
266 cherrypy.request.headers.pop("Content-File-MD5", None)
267 else:
268 error_text = "Invalid yaml format "
269 indata = yaml.safe_load(cherrypy.request.body)
270 cherrypy.request.headers.pop("Content-File-MD5", None)
272 if not indata:
273 indata = {}
275 format_yaml = False
276 if cherrypy.request.headers.get("Query-String-Format") == "yaml":
277 format_yaml = True
279 for k, v in kwargs.items():
280 if isinstance(v, str):
281 if v == "":
282 kwargs[k] = None
283 elif format_yaml:
284 try:
285 kwargs[k] = yaml.safe_load(v)
286 except Exception as yaml_error:
287 logging.exception(
288 f"{yaml_error} occured while parsing the yaml"
289 )
290 elif (
291 k.endswith(".gt")
292 or k.endswith(".lt")
293 or k.endswith(".gte")
294 or k.endswith(".lte")
295 ):
296 try:
297 kwargs[k] = int(v)
298 except Exception:
299 try:
300 kwargs[k] = float(v)
301 except Exception as keyword_error:
302 logging.exception(
303 f"{keyword_error} occured while getting the keyword arguments"
304 )
305 elif v.find(",") > 0:
306 kwargs[k] = v.split(",")
307 elif isinstance(v, (list, tuple)):
308 for index in range(0, len(v)):
309 if v[index] == "":
310 v[index] = None
311 elif format_yaml:
312 try:
313 v[index] = yaml.safe_load(v[index])
314 except Exception as error:
315 logging.exception(
316 f"{error} occured while parsing the yaml"
317 )
319 return indata
320 except (ValueError, yaml.YAMLError) as exc:
321 raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
322 except KeyError as exc:
323 raise RoException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
324 except Exception as exc:
325 raise RoException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
327 @staticmethod
328 def _format_out(data, token_info=None, _format=None):
329 """
330 return string of dictionary data according to requested json, yaml, xml. By default json
331 :param data: response to be sent. Can be a dict, text or file
332 :param token_info: Contains among other username and project
333 :param _format: The format to be set as Content-Type if data is a file
334 :return: None
335 """
336 accept = cherrypy.request.headers.get("Accept")
338 if data is None:
339 if accept and "text/html" in accept:
340 return html.format(
341 data, cherrypy.request, cherrypy.response, token_info
342 )
344 # cherrypy.response.status = HTTPStatus.NO_CONTENT.value
345 return
346 elif hasattr(data, "read"): # file object
347 if _format:
348 cherrypy.response.headers["Content-Type"] = _format
349 elif "b" in data.mode: # binariy asssumig zip
350 cherrypy.response.headers["Content-Type"] = "application/zip"
351 else:
352 cherrypy.response.headers["Content-Type"] = "text/plain"
354 # TODO check that cherrypy close file. If not implement pending things to close per thread next
355 return data
357 if accept:
358 if "application/json" in accept:
359 cherrypy.response.headers["Content-Type"] = (
360 "application/json; charset=utf-8"
361 )
362 a = json.dumps(data, indent=4) + "\n"
364 return a.encode("utf8")
365 elif "text/html" in accept:
366 return html.format(
367 data, cherrypy.request, cherrypy.response, token_info
368 )
369 elif (
370 "application/yaml" in accept
371 or "*/*" in accept
372 or "text/plain" in accept
373 ):
374 pass
375 # if there is not any valid accept, raise an error. But if response is already an error, format in yaml
376 elif cherrypy.response.status >= 400:
377 raise cherrypy.HTTPError(
378 HTTPStatus.NOT_ACCEPTABLE.value,
379 "Only 'Accept' of type 'application/json' or 'application/yaml' "
380 "for output format are available",
381 )
383 cherrypy.response.headers["Content-Type"] = "application/yaml"
385 return yaml.safe_dump(
386 data,
387 explicit_start=True,
388 indent=4,
389 default_flow_style=False,
390 tags=False,
391 encoding="utf-8",
392 allow_unicode=True,
393 ) # , canonical=True, default_style='"'
395 @cherrypy.expose
396 def index(self, *args, **kwargs):
397 token_info = None
399 try:
400 if cherrypy.request.method == "GET":
401 token_info = self.authenticator.authorize()
402 outdata = token_info # Home page
403 else:
404 raise cherrypy.HTTPError(
405 HTTPStatus.METHOD_NOT_ALLOWED.value,
406 "Method {} not allowed for tokens".format(cherrypy.request.method),
407 )
409 return self._format_out(outdata, token_info)
410 except (NsException, AuthException) as e:
411 # cherrypy.log("index Exception {}".format(e))
412 cherrypy.response.status = e.http_code.value
414 return self._format_out("Welcome to OSM!", token_info)
416 @cherrypy.expose
417 def version(self, *args, **kwargs):
418 # TODO consider to remove and provide version using the static version file
419 try:
420 if cherrypy.request.method != "GET":
421 raise RoException(
422 "Only method GET is allowed",
423 HTTPStatus.METHOD_NOT_ALLOWED,
424 )
425 elif args or kwargs:
426 raise RoException(
427 "Invalid URL or query string for version",
428 HTTPStatus.METHOD_NOT_ALLOWED,
429 )
431 # TODO include version of other modules, pick up from some kafka admin message
432 osm_ng_ro_version = {"version": ro_version, "date": ro_version_date}
434 return self._format_out(osm_ng_ro_version)
435 except RoException as e:
436 cherrypy.response.status = e.http_code.value
437 problem_details = {
438 "code": e.http_code.name,
439 "status": e.http_code.value,
440 "detail": str(e),
441 }
443 return self._format_out(problem_details, None)
445 def new_token(self, engine_session, indata, *args, **kwargs):
446 token_info = None
448 try:
449 token_info = self.authenticator.authorize()
450 except Exception:
451 token_info = None
453 if kwargs:
454 indata.update(kwargs)
456 # This is needed to log the user when authentication fails
457 cherrypy.request.login = "{}".format(indata.get("username", "-"))
458 token_info = self.authenticator.new_token(
459 token_info, indata, cherrypy.request.remote
460 )
461 cherrypy.session["Authorization"] = token_info["id"]
462 self._set_location_header("admin", "v1", "tokens", token_info["id"])
463 # for logging
465 # cherrypy.response.cookie["Authorization"] = outdata["id"]
466 # cherrypy.response.cookie["Authorization"]['expires'] = 3600
468 return token_info, token_info["id"], True
470 def del_token(self, engine_session, indata, version, _id, *args, **kwargs):
471 token_id = _id
473 if not token_id and "id" in kwargs:
474 token_id = kwargs["id"]
475 elif not token_id:
476 token_info = self.authenticator.authorize()
477 # for logging
478 token_id = token_info["id"]
480 self.authenticator.del_token(token_id)
481 token_info = None
482 cherrypy.session["Authorization"] = "logout"
483 # cherrypy.response.cookie["Authorization"] = token_id
484 # cherrypy.response.cookie["Authorization"]['expires'] = 0
486 return None, None, True
488 @cherrypy.expose
489 def test(self, *args, **kwargs):
490 if not cherrypy.config.get("server.enable_test") or (
491 isinstance(cherrypy.config["server.enable_test"], str)
492 and cherrypy.config["server.enable_test"].lower() == "false"
493 ):
494 cherrypy.response.status = HTTPStatus.METHOD_NOT_ALLOWED.value
496 return "test URL is disabled"
498 thread_info = None
500 if args and args[0] == "help":
501 return (
502 "<html><pre>\ninit\nfile/<name> download file\ndb-clear/table\nfs-clear[/folder]\nlogin\nlogin2\n"
503 "sleep/<time>\nmessage/topic\n</pre></html>"
504 )
505 elif args and args[0] == "init":
506 try:
507 # self.ns.load_dbase(cherrypy.request.app.config)
508 self.ns.create_admin()
510 return "Done. User 'admin', password 'admin' created"
511 except Exception:
512 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
514 return self._format_out("Database already initialized")
515 elif args and args[0] == "file":
516 return cherrypy.lib.static.serve_file(
517 cherrypy.tree.apps["/ro"].config["storage"]["path"] + "/" + args[1],
518 "text/plain",
519 "attachment",
520 )
521 elif args and args[0] == "file2":
522 f_path = cherrypy.tree.apps["/ro"].config["storage"]["path"] + "/" + args[1]
523 f = open(f_path, "r")
524 cherrypy.response.headers["Content-type"] = "text/plain"
525 return f
527 elif len(args) == 2 and args[0] == "db-clear":
528 deleted_info = self.ns.db.del_list(args[1], kwargs)
529 return "{} {} deleted\n".format(deleted_info["deleted"], args[1])
530 elif len(args) and args[0] == "fs-clear":
531 if len(args) >= 2:
532 folders = (args[1],)
533 else:
534 folders = self.ns.fs.dir_ls(".")
536 for folder in folders:
537 self.ns.fs.file_delete(folder)
539 return ",".join(folders) + " folders deleted\n"
540 elif args and args[0] == "login":
541 if not cherrypy.request.headers.get("Authorization"):
542 cherrypy.response.headers["WWW-Authenticate"] = (
543 'Basic realm="Access to OSM site", charset="UTF-8"'
544 )
545 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
546 elif args and args[0] == "login2":
547 if not cherrypy.request.headers.get("Authorization"):
548 cherrypy.response.headers["WWW-Authenticate"] = (
549 'Bearer realm="Access to OSM site"'
550 )
551 cherrypy.response.status = HTTPStatus.UNAUTHORIZED.value
552 elif args and args[0] == "sleep":
553 sleep_time = 5
555 try:
556 sleep_time = int(args[1])
557 except Exception:
558 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
559 return self._format_out("Database already initialized")
561 thread_info = cherrypy.thread_data
562 print(thread_info)
563 time.sleep(sleep_time)
564 # thread_info
565 elif len(args) >= 2 and args[0] == "message":
566 main_topic = args[1]
567 return_text = "<html><pre>{} ->\n".format(main_topic)
569 try:
570 if cherrypy.request.method == "POST":
571 to_send = yaml.safe_load(cherrypy.request.body)
572 for k, v in to_send.items():
573 self.ns.msg.write(main_topic, k, v)
574 return_text += " {}: {}\n".format(k, v)
575 elif cherrypy.request.method == "GET":
576 for k, v in kwargs.items():
577 self.ns.msg.write(main_topic, k, yaml.safe_load(v))
578 return_text += " {}: {}\n".format(k, yaml.safe_load(v))
579 except Exception as e:
580 return_text += "Error: " + str(e)
582 return_text += "</pre></html>\n"
584 return return_text
586 return_text = (
587 "<html><pre>\nheaders:\n args: {}\n".format(args)
588 + " kwargs: {}\n".format(kwargs)
589 + " headers: {}\n".format(cherrypy.request.headers)
590 + " path_info: {}\n".format(cherrypy.request.path_info)
591 + " query_string: {}\n".format(cherrypy.request.query_string)
592 + " session: {}\n".format(cherrypy.session)
593 + " cookie: {}\n".format(cherrypy.request.cookie)
594 + " method: {}\n".format(cherrypy.request.method)
595 + " session: {}\n".format(cherrypy.session.get("fieldname"))
596 + " body:\n"
597 )
598 return_text += " length: {}\n".format(cherrypy.request.body.length)
600 if cherrypy.request.body.length:
601 return_text += " content: {}\n".format(
602 str(
603 cherrypy.request.body.read(
604 int(cherrypy.request.headers.get("Content-Length", 0))
605 )
606 )
607 )
609 if thread_info:
610 return_text += "thread: {}\n".format(thread_info)
612 return_text += "</pre></html>"
614 return return_text
616 @staticmethod
617 def _check_valid_url_method(method, *args):
618 if len(args) < 3:
619 raise RoException(
620 "URL must contain at least 'main_topic/version/topic'",
621 HTTPStatus.METHOD_NOT_ALLOWED,
622 )
624 reference = valid_url_methods
625 for arg in args:
626 if arg is None:
627 break
629 if not isinstance(reference, dict):
630 raise RoException(
631 "URL contains unexpected extra items '{}'".format(arg),
632 HTTPStatus.METHOD_NOT_ALLOWED,
633 )
635 if arg in reference:
636 reference = reference[arg]
637 elif "<ID>" in reference:
638 reference = reference["<ID>"]
639 elif "*" in reference:
640 # reference = reference["*"]
641 break
642 else:
643 raise RoException(
644 "Unexpected URL item {}".format(arg),
645 HTTPStatus.METHOD_NOT_ALLOWED,
646 )
648 if "TODO" in reference and method in reference["TODO"]:
649 raise RoException(
650 "Method {} not supported yet for this URL".format(method),
651 HTTPStatus.NOT_IMPLEMENTED,
652 )
653 elif "METHODS" not in reference or method not in reference["METHODS"]:
654 raise RoException(
655 "Method {} not supported for this URL".format(method),
656 HTTPStatus.METHOD_NOT_ALLOWED,
657 )
659 return reference["ROLE_PERMISSION"] + method.lower()
661 @staticmethod
662 def _set_location_header(main_topic, version, topic, id):
663 """
664 Insert response header Location with the URL of created item base on URL params
665 :param main_topic:
666 :param version:
667 :param topic:
668 :param id:
669 :return: None
670 """
671 # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
672 cherrypy.response.headers["Location"] = "/ro/{}/{}/{}/{}".format(
673 main_topic, version, topic, id
674 )
676 return
678 @cherrypy.expose
679 def default(
680 self,
681 main_topic=None,
682 version=None,
683 topic=None,
684 _id=None,
685 _id2=None,
686 *args,
687 **kwargs,
688 ):
689 token_info = None
690 outdata = {}
691 _format = None
692 method = "DONE"
693 rollback = []
694 engine_session = None
696 try:
697 if not main_topic or not version or not topic:
698 raise RoException(
699 "URL must contain at least 'main_topic/version/topic'",
700 HTTPStatus.METHOD_NOT_ALLOWED,
701 )
703 if main_topic not in (
704 "admin",
705 "ns",
706 ):
707 raise RoException(
708 "URL main_topic '{}' not supported".format(main_topic),
709 HTTPStatus.METHOD_NOT_ALLOWED,
710 )
712 if version != "v1":
713 raise RoException(
714 "URL version '{}' not supported".format(version),
715 HTTPStatus.METHOD_NOT_ALLOWED,
716 )
718 if (
719 kwargs
720 and "METHOD" in kwargs
721 and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH")
722 ):
723 method = kwargs.pop("METHOD")
724 else:
725 method = cherrypy.request.method
727 role_permission = self._check_valid_url_method(
728 method, main_topic, version, topic, _id, _id2, *args, **kwargs
729 )
730 # skip token validation if requesting a token
731 indata = self._format_in(kwargs)
733 if main_topic != "admin" or topic != "tokens":
734 token_info = self.authenticator.authorize(role_permission, _id)
736 outdata, created_id, done = self.map_operation[role_permission](
737 engine_session, indata, version, _id, _id2, *args, *kwargs
738 )
740 if created_id:
741 self._set_location_header(main_topic, version, topic, _id)
743 cherrypy.response.status = (
744 HTTPStatus.ACCEPTED.value
745 if not done
746 else (
747 HTTPStatus.OK.value
748 if outdata is not None
749 else HTTPStatus.NO_CONTENT.value
750 )
751 )
753 return self._format_out(outdata, token_info, _format)
754 except Exception as e:
755 if isinstance(
756 e,
757 (
758 RoException,
759 NsException,
760 DbException,
761 FsException,
762 MsgException,
763 AuthException,
764 ValidationError,
765 ),
766 ):
767 http_code_value = cherrypy.response.status = e.http_code.value
768 http_code_name = e.http_code.name
769 cherrypy.log("Exception {}".format(e))
770 else:
771 http_code_value = cherrypy.response.status = (
772 HTTPStatus.BAD_REQUEST.value
773 ) # INTERNAL_SERVER_ERROR
774 cherrypy.log("CRITICAL: Exception {}".format(e), traceback=True)
775 http_code_name = HTTPStatus.BAD_REQUEST.name
777 if hasattr(outdata, "close"): # is an open file
778 outdata.close()
780 error_text = str(e)
781 rollback.reverse()
783 for rollback_item in rollback:
784 try:
785 if rollback_item.get("operation") == "set":
786 self.ns.db.set_one(
787 rollback_item["topic"],
788 {"_id": rollback_item["_id"]},
789 rollback_item["content"],
790 fail_on_empty=False,
791 )
792 else:
793 self.ns.db.del_one(
794 rollback_item["topic"],
795 {"_id": rollback_item["_id"]},
796 fail_on_empty=False,
797 )
798 except Exception as e2:
799 rollback_error_text = "Rollback Exception {}: {}".format(
800 rollback_item, e2
801 )
802 cherrypy.log(rollback_error_text)
803 error_text += ". " + rollback_error_text
805 # if isinstance(e, MsgException):
806 # error_text = "{} has been '{}' but other modules cannot be informed because an error on bus".format(
807 # engine_topic[:-1], method, error_text)
808 problem_details = {
809 "code": http_code_name,
810 "status": http_code_value,
811 "detail": error_text,
812 }
814 return self._format_out(problem_details, token_info)
815 # raise cherrypy.HTTPError(e.http_code.value, str(e))
816 finally:
817 if token_info:
818 if method in ("PUT", "PATCH", "POST") and isinstance(outdata, dict):
819 for logging_id in ("id", "op_id", "nsilcmop_id", "nslcmop_id"):
820 if outdata.get(logging_id):
821 cherrypy.request.login += ";{}={}".format(
822 logging_id, outdata[logging_id][:36]
823 )
826def _start_service():
827 """
828 Callback function called when cherrypy.engine starts
829 Override configuration with env variables
830 Set database, storage, message configuration
831 Init database with admin/admin user password
832 """
833 global ro_server, vim_admin_thread
834 # global vim_threads
835 cherrypy.log.error("Starting osm_ng_ro")
836 # update general cherrypy configuration
837 update_dict = {}
838 engine_config = cherrypy.tree.apps["/ro"].config
840 for k, v in environ.items():
841 if not k.startswith("OSMRO_"):
842 continue
844 k1, _, k2 = k[6:].lower().partition("_")
846 if not k2:
847 continue
849 try:
850 if k1 in ("server", "test", "auth", "log"):
851 # update [global] configuration
852 update_dict[k1 + "." + k2] = yaml.safe_load(v)
853 elif k1 == "static":
854 # update [/static] configuration
855 engine_config["/static"]["tools.staticdir." + k2] = yaml.safe_load(v)
856 elif k1 == "tools":
857 # update [/] configuration
858 engine_config["/"]["tools." + k2.replace("_", ".")] = yaml.safe_load(v)
859 elif k1 in ("message", "database", "storage", "authentication", "period"):
860 engine_config[k1][k2] = yaml.safe_load(v)
862 except Exception as e:
863 raise RoException("Cannot load env '{}': {}".format(k, e))
865 if update_dict:
866 cherrypy.config.update(update_dict)
867 engine_config["global"].update(update_dict)
869 # logging cherrypy
870 log_format_simple = (
871 "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
872 )
873 log_formatter_simple = logging.Formatter(
874 log_format_simple, datefmt="%Y-%m-%dT%H:%M:%S"
875 )
876 logger_server = logging.getLogger("cherrypy.error")
877 logger_access = logging.getLogger("cherrypy.access")
878 logger_cherry = logging.getLogger("cherrypy")
879 logger = logging.getLogger("ro")
881 if "log.file" in engine_config["global"]:
882 file_handler = logging.handlers.RotatingFileHandler(
883 engine_config["global"]["log.file"], maxBytes=100e6, backupCount=9, delay=0
884 )
885 file_handler.setFormatter(log_formatter_simple)
886 logger_cherry.addHandler(file_handler)
887 logger.addHandler(file_handler)
889 # log always to standard output
890 for format_, logger in {
891 "ro.server %(filename)s:%(lineno)s": logger_server,
892 "ro.access %(filename)s:%(lineno)s": logger_access,
893 "%(name)s %(filename)s:%(lineno)s": logger,
894 }.items():
895 log_format_cherry = "%(asctime)s %(levelname)s {} %(message)s".format(format_)
896 log_formatter_cherry = logging.Formatter(
897 log_format_cherry, datefmt="%Y-%m-%dT%H:%M:%S"
898 )
899 str_handler = logging.StreamHandler()
900 str_handler.setFormatter(log_formatter_cherry)
901 logger.addHandler(str_handler)
903 if engine_config["global"].get("log.level"):
904 logger_cherry.setLevel(engine_config["global"]["log.level"])
905 logger.setLevel(engine_config["global"]["log.level"])
907 # logging other modules
908 for k1, logname in {
909 "message": "ro.msg",
910 "database": "ro.db",
911 "storage": "ro.fs",
912 }.items():
913 engine_config[k1]["logger_name"] = logname
914 logger_module = logging.getLogger(logname)
916 if "logfile" in engine_config[k1]:
917 file_handler = logging.handlers.RotatingFileHandler(
918 engine_config[k1]["logfile"], maxBytes=100e6, backupCount=9, delay=0
919 )
920 file_handler.setFormatter(log_formatter_simple)
921 logger_module.addHandler(file_handler)
923 if "loglevel" in engine_config[k1]:
924 logger_module.setLevel(engine_config[k1]["loglevel"])
925 # TODO add more entries, e.g.: storage
927 engine_config["assignment"] = {}
928 # ^ each VIM, SDNc will be assigned one worker id. Ns class will add items and VimThread will auto-assign
929 cherrypy.tree.apps["/ro"].root.ns.start(engine_config)
930 cherrypy.tree.apps["/ro"].root.authenticator.start(engine_config)
931 cherrypy.tree.apps["/ro"].root.ns.init_db(target_version=database_version)
933 # # start subscriptions thread:
934 vim_admin_thread = VimAdminThread(config=engine_config, engine=ro_server.ns)
935 vim_admin_thread.start()
936 start_monitoring(config=engine_config)
938 # # Do not capture except SubscriptionException
940 # backend = engine_config["authentication"]["backend"]
941 # cherrypy.log.error("Starting OSM NBI Version '{} {}' with '{}' authentication backend"
942 # .format(ro_version, ro_version_date, backend))
945def _stop_service():
946 """
947 Callback function called when cherrypy.engine stops
948 TODO: Ending database connections.
949 """
950 global vim_admin_thread
952 # terminate vim_admin_thread
953 if vim_admin_thread:
954 vim_admin_thread.terminate()
955 stop_monitoring()
956 vim_admin_thread = None
957 cherrypy.tree.apps["/ro"].root.ns.stop()
958 cherrypy.log.error("Stopping osm_ng_ro")
961def ro_main(config_file):
962 global ro_server
964 ro_server = Server()
965 cherrypy.engine.subscribe("start", _start_service)
966 cherrypy.engine.subscribe("stop", _stop_service)
967 cherrypy.quickstart(ro_server, "/ro", config_file)
970def usage():
971 print(
972 """Usage: {} [options]
973 -c|--config [configuration_file]: loads the configuration file (default: ./ro.cfg)
974 -h|--help: shows this help
975 """.format(
976 sys.argv[0]
977 )
978 )
979 # --log-socket-host HOST: send logs to this host")
980 # --log-socket-port PORT: send logs using this port (default: 9022)")
983if __name__ == "__main__":
984 try:
985 # load parameters and configuration
986 opts, args = getopt.getopt(sys.argv[1:], "hvc:", ["config=", "help"])
987 # TODO add "log-socket-host=", "log-socket-port=", "log-file="
988 config_file = None
990 for o, a in opts:
991 if o in ("-h", "--help"):
992 usage()
993 sys.exit()
994 elif o in ("-c", "--config"):
995 config_file = a
996 else:
997 raise ValueError("Unhandled option")
999 if config_file:
1000 if not path.isfile(config_file):
1001 print(
1002 "configuration file '{}' that not exist".format(config_file),
1003 file=sys.stderr,
1004 )
1005 exit(1)
1006 else:
1007 for config_file in (
1008 path.dirname(__file__) + "/ro.cfg",
1009 "./ro.cfg",
1010 "/etc/osm/ro.cfg",
1011 ):
1012 if path.isfile(config_file):
1013 break
1014 else:
1015 print(
1016 "No configuration file 'ro.cfg' found neither at local folder nor at /etc/osm/",
1017 file=sys.stderr,
1018 )
1019 exit(1)
1021 ro_main(config_file)
1022 except KeyboardInterrupt:
1023 print("KeyboardInterrupt. Finishing", file=sys.stderr)
1024 except getopt.GetoptError as e:
1025 print(str(e), file=sys.stderr)
1026 # usage()
1027 exit(1)