blob: b306720246b9e4fa72c90ae522c8a0fb6a191f12 [file] [log] [blame]
tiernoc0e42e22018-05-11 11:36:10 +02001#!/usr/bin/python3
2# -*- coding: utf-8 -*-
3
tierno2e215512018-11-28 09:37:52 +00004##
5# Copyright 2018 Telefonica S.A.
6#
7# Licensed under the Apache License, Version 2.0 (the "License"); you may
8# not use this file except in compliance with the License. You may obtain
9# a copy of the License at
10#
11# http://www.apache.org/licenses/LICENSE-2.0
12#
13# Unless required by applicable law or agreed to in writing, software
14# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
15# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
16# License for the specific language governing permissions and limitations
17# under the License.
18##
19
quilesj7e13aeb2019-10-08 13:34:55 +020020
21# DEBUG WITH PDB
22import os
23import pdb
24
tiernoc0e42e22018-05-11 11:36:10 +020025import asyncio
26import yaml
tierno275411e2018-05-16 14:33:32 +020027import logging
28import logging.handlers
29import getopt
tierno275411e2018-05-16 14:33:32 +020030import sys
tierno59d22d22018-09-25 18:10:19 +020031
quilesj7e13aeb2019-10-08 13:34:55 +020032from osm_lcm import ns
33from osm_lcm import vim_sdn
34from osm_lcm import netslice
35from osm_lcm import ROclient
36
tierno94f06112020-02-11 12:38:19 +000037from time import time
tierno8069ce52019-08-28 15:34:33 +000038from osm_lcm.lcm_utils import versiontuple, LcmException, TaskRegistry, LcmExceptionExit
tiernoa4dea5a2020-01-05 16:29:30 +000039from osm_lcm import version as lcm_version, version_date as lcm_version_date
tierno8069ce52019-08-28 15:34:33 +000040
Eduardo Sousa5899e082019-06-21 11:54:56 +010041from osm_common import dbmemory, dbmongo, fslocal, fsmongo, msglocal, msgkafka
tierno98768132018-09-11 12:07:21 +020042from osm_common import version as common_version
tierno59d22d22018-09-25 18:10:19 +020043from osm_common.dbbase import DbException
tiernoc0e42e22018-05-11 11:36:10 +020044from osm_common.fsbase import FsException
45from osm_common.msgbase import MsgException
tierno275411e2018-05-16 14:33:32 +020046from os import environ, path
tierno16427352019-04-22 11:37:36 +000047from random import choice as random_choice
tierno59d22d22018-09-25 18:10:19 +020048from n2vc import version as n2vc_version
tiernoc0e42e22018-05-11 11:36:10 +020049
quilesj7e13aeb2019-10-08 13:34:55 +020050if os.getenv('OSMLCM_PDB_DEBUG', None) is not None:
51 pdb.set_trace()
52
tiernoc0e42e22018-05-11 11:36:10 +020053
tierno275411e2018-05-16 14:33:32 +020054__author__ = "Alfonso Tierno"
tiernoe64f7fb2019-09-11 08:55:52 +000055min_RO_version = "6.0.2"
tierno6e9d2eb2018-09-12 17:47:18 +020056min_n2vc_version = "0.0.2"
quilesj7e13aeb2019-10-08 13:34:55 +020057
tierno16427352019-04-22 11:37:36 +000058min_common_version = "0.1.19"
tierno3e359b12019-02-03 02:29:13 +010059health_check_file = path.expanduser("~") + "/time_last_ping" # TODO find better location for this file
tierno275411e2018-05-16 14:33:32 +020060
61
tiernoc0e42e22018-05-11 11:36:10 +020062class Lcm:
63
tiernoa9843d82018-10-24 10:44:20 +020064 ping_interval_pace = 120 # how many time ping is send once is confirmed all is running
tiernof578e552018-11-08 19:07:20 +010065 ping_interval_boot = 5 # how many time ping is sent when booting
tiernoa9843d82018-10-24 10:44:20 +020066
tierno59d22d22018-09-25 18:10:19 +020067 def __init__(self, config_file, loop=None):
tiernoc0e42e22018-05-11 11:36:10 +020068 """
69 Init, Connect to database, filesystem storage, and messaging
70 :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
71 :return: None
72 """
tiernoc0e42e22018-05-11 11:36:10 +020073 self.db = None
74 self.msg = None
tierno16427352019-04-22 11:37:36 +000075 self.msg_admin = None
tiernoc0e42e22018-05-11 11:36:10 +020076 self.fs = None
77 self.pings_not_received = 1
tiernoc2564fe2019-01-28 16:18:56 +000078 self.consecutive_errors = 0
79 self.first_start = False
tiernoc0e42e22018-05-11 11:36:10 +020080
tiernoc0e42e22018-05-11 11:36:10 +020081 # logging
82 self.logger = logging.getLogger('lcm')
tierno16427352019-04-22 11:37:36 +000083 # get id
84 self.worker_id = self.get_process_id()
tiernoc0e42e22018-05-11 11:36:10 +020085 # load configuration
86 config = self.read_config_file(config_file)
87 self.config = config
tierno744303e2020-01-13 16:46:31 +000088 self.config["ro_config"] = {
tiernoc0e42e22018-05-11 11:36:10 +020089 "endpoint_url": "http://{}:{}/openmano".format(config["RO"]["host"], config["RO"]["port"]),
tierno750b2452018-05-17 16:39:29 +020090 "tenant": config.get("tenant", "osm"),
tiernoc0e42e22018-05-11 11:36:10 +020091 "logger_name": "lcm.ROclient",
92 "loglevel": "ERROR",
93 }
94
tierno59d22d22018-09-25 18:10:19 +020095 self.loop = loop or asyncio.get_event_loop()
tiernoc0e42e22018-05-11 11:36:10 +020096
97 # logging
98 log_format_simple = "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s"
99 log_formatter_simple = logging.Formatter(log_format_simple, datefmt='%Y-%m-%dT%H:%M:%S')
100 config["database"]["logger_name"] = "lcm.db"
101 config["storage"]["logger_name"] = "lcm.fs"
102 config["message"]["logger_name"] = "lcm.msg"
tierno86aa62f2018-08-20 11:57:04 +0000103 if config["global"].get("logfile"):
tiernoc0e42e22018-05-11 11:36:10 +0200104 file_handler = logging.handlers.RotatingFileHandler(config["global"]["logfile"],
105 maxBytes=100e6, backupCount=9, delay=0)
106 file_handler.setFormatter(log_formatter_simple)
107 self.logger.addHandler(file_handler)
tierno86aa62f2018-08-20 11:57:04 +0000108 if not config["global"].get("nologging"):
tiernoc0e42e22018-05-11 11:36:10 +0200109 str_handler = logging.StreamHandler()
110 str_handler.setFormatter(log_formatter_simple)
111 self.logger.addHandler(str_handler)
112
113 if config["global"].get("loglevel"):
114 self.logger.setLevel(config["global"]["loglevel"])
115
116 # logging other modules
117 for k1, logname in {"message": "lcm.msg", "database": "lcm.db", "storage": "lcm.fs"}.items():
118 config[k1]["logger_name"] = logname
119 logger_module = logging.getLogger(logname)
tierno86aa62f2018-08-20 11:57:04 +0000120 if config[k1].get("logfile"):
tiernoc0e42e22018-05-11 11:36:10 +0200121 file_handler = logging.handlers.RotatingFileHandler(config[k1]["logfile"],
122 maxBytes=100e6, backupCount=9, delay=0)
123 file_handler.setFormatter(log_formatter_simple)
124 logger_module.addHandler(file_handler)
tierno86aa62f2018-08-20 11:57:04 +0000125 if config[k1].get("loglevel"):
tiernoc0e42e22018-05-11 11:36:10 +0200126 logger_module.setLevel(config[k1]["loglevel"])
tierno86aa62f2018-08-20 11:57:04 +0000127 self.logger.critical("starting osm/lcm version {} {}".format(lcm_version, lcm_version_date))
tierno59d22d22018-09-25 18:10:19 +0200128
tiernoc0e42e22018-05-11 11:36:10 +0200129 # check version of N2VC
130 # TODO enhance with int conversion or from distutils.version import LooseVersion
131 # or with list(map(int, version.split(".")))
tierno59d22d22018-09-25 18:10:19 +0200132 if versiontuple(n2vc_version) < versiontuple(min_n2vc_version):
tierno6e9d2eb2018-09-12 17:47:18 +0200133 raise LcmException("Not compatible osm/N2VC version '{}'. Needed '{}' or higher".format(
tierno59d22d22018-09-25 18:10:19 +0200134 n2vc_version, min_n2vc_version))
135 # check version of common
tierno27246d82018-09-27 15:59:09 +0200136 if versiontuple(common_version) < versiontuple(min_common_version):
tierno6e9d2eb2018-09-12 17:47:18 +0200137 raise LcmException("Not compatible osm/common version '{}'. Needed '{}' or higher".format(
138 common_version, min_common_version))
tierno22f4f9c2018-06-11 18:53:39 +0200139
tiernoc0e42e22018-05-11 11:36:10 +0200140 try:
tierno22f4f9c2018-06-11 18:53:39 +0200141 # TODO check database version
tiernoc0e42e22018-05-11 11:36:10 +0200142 if config["database"]["driver"] == "mongo":
143 self.db = dbmongo.DbMongo()
144 self.db.db_connect(config["database"])
145 elif config["database"]["driver"] == "memory":
146 self.db = dbmemory.DbMemory()
147 self.db.db_connect(config["database"])
148 else:
149 raise LcmException("Invalid configuration param '{}' at '[database]':'driver'".format(
150 config["database"]["driver"]))
151
152 if config["storage"]["driver"] == "local":
153 self.fs = fslocal.FsLocal()
154 self.fs.fs_connect(config["storage"])
Eduardo Sousa5899e082019-06-21 11:54:56 +0100155 elif config["storage"]["driver"] == "mongo":
156 self.fs = fsmongo.FsMongo()
157 self.fs.fs_connect(config["storage"])
tiernoc0e42e22018-05-11 11:36:10 +0200158 else:
159 raise LcmException("Invalid configuration param '{}' at '[storage]':'driver'".format(
160 config["storage"]["driver"]))
161
quilesj7e13aeb2019-10-08 13:34:55 +0200162 # copy message configuration in order to remove 'group_id' for msg_admin
tiernoc2564fe2019-01-28 16:18:56 +0000163 config_message = config["message"].copy()
164 config_message["loop"] = self.loop
165 if config_message["driver"] == "local":
tiernoc0e42e22018-05-11 11:36:10 +0200166 self.msg = msglocal.MsgLocal()
tiernoc2564fe2019-01-28 16:18:56 +0000167 self.msg.connect(config_message)
tierno16427352019-04-22 11:37:36 +0000168 self.msg_admin = msglocal.MsgLocal()
169 config_message.pop("group_id", None)
170 self.msg_admin.connect(config_message)
tiernoc2564fe2019-01-28 16:18:56 +0000171 elif config_message["driver"] == "kafka":
tiernoc0e42e22018-05-11 11:36:10 +0200172 self.msg = msgkafka.MsgKafka()
tiernoc2564fe2019-01-28 16:18:56 +0000173 self.msg.connect(config_message)
tierno16427352019-04-22 11:37:36 +0000174 self.msg_admin = msgkafka.MsgKafka()
175 config_message.pop("group_id", None)
176 self.msg_admin.connect(config_message)
tiernoc0e42e22018-05-11 11:36:10 +0200177 else:
178 raise LcmException("Invalid configuration param '{}' at '[message]':'driver'".format(
tiernoc2564fe2019-01-28 16:18:56 +0000179 config["message"]["driver"]))
tiernoc0e42e22018-05-11 11:36:10 +0200180 except (DbException, FsException, MsgException) as e:
181 self.logger.critical(str(e), exc_info=True)
182 raise LcmException(str(e))
183
kuused124bfe2019-06-18 12:09:24 +0200184 # contains created tasks/futures to be able to cancel
185 self.lcm_tasks = TaskRegistry(self.worker_id, self.db, self.logger)
186
tierno744303e2020-01-13 16:46:31 +0000187 self.ns = ns.NsLcm(self.db, self.msg, self.fs, self.lcm_tasks, self.config, self.loop)
188 self.netslice = netslice.NetsliceLcm(self.db, self.msg, self.fs, self.lcm_tasks, self.config, self.loop)
189 self.vim = vim_sdn.VimLcm(self.db, self.msg, self.fs, self.lcm_tasks, self.config, self.loop)
190 self.wim = vim_sdn.WimLcm(self.db, self.msg, self.fs, self.lcm_tasks, self.config, self.loop)
191 self.sdn = vim_sdn.SdnLcm(self.db, self.msg, self.fs, self.lcm_tasks, self.config, self.loop)
192 self.k8scluster = vim_sdn.K8sClusterLcm(self.db, self.msg, self.fs, self.lcm_tasks, self.config, self.loop)
193 self.k8srepo = vim_sdn.K8sRepoLcm(self.db, self.msg, self.fs, self.lcm_tasks, self.config, self.loop)
tierno59d22d22018-09-25 18:10:19 +0200194
tierno22f4f9c2018-06-11 18:53:39 +0200195 async def check_RO_version(self):
tiernoe64f7fb2019-09-11 08:55:52 +0000196 tries = 14
197 last_error = None
198 while True:
199 try:
tierno744303e2020-01-13 16:46:31 +0000200 ro_server = ROclient.ROClient(self.loop, **self.config["ro_config"])
tiernoe64f7fb2019-09-11 08:55:52 +0000201 ro_version = await ro_server.get_version()
202 if versiontuple(ro_version) < versiontuple(min_RO_version):
203 raise LcmException("Not compatible osm/RO version '{}'. Needed '{}' or higher".format(
204 ro_version, min_RO_version))
205 self.logger.info("Connected to RO version {}".format(ro_version))
206 return
207 except ROclient.ROClientException as e:
208 tries -= 1
tierno744303e2020-01-13 16:46:31 +0000209 error_text = "Error while connecting to RO on {}: {}".format(self.config["ro_config"]["endpoint_url"],
210 e)
tiernoe64f7fb2019-09-11 08:55:52 +0000211 if tries <= 0:
212 self.logger.critical(error_text)
213 raise LcmException(error_text)
214 if last_error != error_text:
215 last_error = error_text
216 self.logger.error(error_text + ". Waiting until {} seconds".format(5*tries))
217 await asyncio.sleep(5)
tierno22f4f9c2018-06-11 18:53:39 +0200218
tiernoc0e42e22018-05-11 11:36:10 +0200219 async def test(self, param=None):
220 self.logger.debug("Starting/Ending test task: {}".format(param))
221
tiernoc0e42e22018-05-11 11:36:10 +0200222 async def kafka_ping(self):
223 self.logger.debug("Task kafka_ping Enter")
224 consecutive_errors = 0
225 first_start = True
226 kafka_has_received = False
227 self.pings_not_received = 1
228 while True:
229 try:
tierno16427352019-04-22 11:37:36 +0000230 await self.msg_admin.aiowrite(
231 "admin", "ping",
232 {"from": "lcm", "to": "lcm", "worker_id": self.worker_id, "version": lcm_version},
233 self.loop)
tiernoc0e42e22018-05-11 11:36:10 +0200234 # time between pings are low when it is not received and at starting
tiernoa9843d82018-10-24 10:44:20 +0200235 wait_time = self.ping_interval_boot if not kafka_has_received else self.ping_interval_pace
tiernoc0e42e22018-05-11 11:36:10 +0200236 if not self.pings_not_received:
237 kafka_has_received = True
238 self.pings_not_received += 1
239 await asyncio.sleep(wait_time, loop=self.loop)
240 if self.pings_not_received > 10:
241 raise LcmException("It is not receiving pings from Kafka bus")
242 consecutive_errors = 0
243 first_start = False
244 except LcmException:
245 raise
246 except Exception as e:
247 # if not first_start is the first time after starting. So leave more time and wait
248 # to allow kafka starts
249 if consecutive_errors == 8 if not first_start else 30:
250 self.logger.error("Task kafka_read task exit error too many errors. Exception: {}".format(e))
251 raise
252 consecutive_errors += 1
253 self.logger.error("Task kafka_read retrying after Exception {}".format(e))
tierno16427352019-04-22 11:37:36 +0000254 wait_time = 2 if not first_start else 5
tiernoc0e42e22018-05-11 11:36:10 +0200255 await asyncio.sleep(wait_time, loop=self.loop)
256
gcalvinoed7f6d42018-12-14 14:44:56 +0100257 def kafka_read_callback(self, topic, command, params):
258 order_id = 1
259
260 if topic != "admin" and command != "ping":
261 self.logger.debug("Task kafka_read receives {} {}: {}".format(topic, command, params))
262 self.consecutive_errors = 0
263 self.first_start = False
264 order_id += 1
265 if command == "exit":
266 raise LcmExceptionExit
267 elif command.startswith("#"):
268 return
269 elif command == "echo":
270 # just for test
271 print(params)
272 sys.stdout.flush()
273 return
274 elif command == "test":
275 asyncio.Task(self.test(params), loop=self.loop)
276 return
277
278 if topic == "admin":
279 if command == "ping" and params["to"] == "lcm" and params["from"] == "lcm":
tierno16427352019-04-22 11:37:36 +0000280 if params.get("worker_id") != self.worker_id:
281 return
gcalvinoed7f6d42018-12-14 14:44:56 +0100282 self.pings_not_received = 0
tierno3e359b12019-02-03 02:29:13 +0100283 try:
284 with open(health_check_file, "w") as f:
285 f.write(str(time()))
286 except Exception as e:
287 self.logger.error("Cannot write into '{}' for healthcheck: {}".format(health_check_file, e))
gcalvinoed7f6d42018-12-14 14:44:56 +0100288 return
magnussonle9198bb2020-01-21 13:00:51 +0100289 elif topic == "pla":
290 if command == "placement":
291 self.ns.update_nsrs_with_pla_result(params)
292 return
calvinosanch9f9c6f22019-11-04 13:37:39 +0100293 elif topic == "k8scluster":
294 if command == "create" or command == "created":
295 k8scluster_id = params.get("_id")
296 task = asyncio.ensure_future(self.k8scluster.create(params, order_id))
297 self.lcm_tasks.register("k8scluster", k8scluster_id, order_id, "k8scluster_create", task)
298 return
299 elif command == "delete" or command == "deleted":
300 k8scluster_id = params.get("_id")
301 task = asyncio.ensure_future(self.k8scluster.delete(params, order_id))
302 self.lcm_tasks.register("k8scluster", k8scluster_id, order_id, "k8scluster_delete", task)
303 return
304 elif topic == "k8srepo":
305 if command == "create" or command == "created":
306 k8srepo_id = params.get("_id")
307 self.logger.debug("k8srepo_id = {}".format(k8srepo_id))
308 task = asyncio.ensure_future(self.k8srepo.create(params, order_id))
309 self.lcm_tasks.register("k8srepo", k8srepo_id, order_id, "k8srepo_create", task)
310 return
311 elif command == "delete" or command == "deleted":
312 k8srepo_id = params.get("_id")
313 task = asyncio.ensure_future(self.k8srepo.delete(params, order_id))
314 self.lcm_tasks.register("k8srepo", k8srepo_id, order_id, "k8srepo_delete", task)
315 return
gcalvinoed7f6d42018-12-14 14:44:56 +0100316 elif topic == "ns":
tierno307425f2020-01-26 23:35:59 +0000317 if command == "instantiate":
gcalvinoed7f6d42018-12-14 14:44:56 +0100318 # self.logger.debug("Deploying NS {}".format(nsr_id))
319 nslcmop = params
320 nslcmop_id = nslcmop["_id"]
321 nsr_id = nslcmop["nsInstanceId"]
322 task = asyncio.ensure_future(self.ns.instantiate(nsr_id, nslcmop_id))
323 self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "ns_instantiate", task)
324 return
tierno307425f2020-01-26 23:35:59 +0000325 elif command == "terminate":
gcalvinoed7f6d42018-12-14 14:44:56 +0100326 # self.logger.debug("Deleting NS {}".format(nsr_id))
327 nslcmop = params
328 nslcmop_id = nslcmop["_id"]
329 nsr_id = nslcmop["nsInstanceId"]
330 self.lcm_tasks.cancel(topic, nsr_id)
331 task = asyncio.ensure_future(self.ns.terminate(nsr_id, nslcmop_id))
332 self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "ns_terminate", task)
333 return
334 elif command == "action":
335 # self.logger.debug("Update NS {}".format(nsr_id))
336 nslcmop = params
337 nslcmop_id = nslcmop["_id"]
338 nsr_id = nslcmop["nsInstanceId"]
339 task = asyncio.ensure_future(self.ns.action(nsr_id, nslcmop_id))
340 self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "ns_action", task)
341 return
342 elif command == "scale":
343 # self.logger.debug("Update NS {}".format(nsr_id))
344 nslcmop = params
345 nslcmop_id = nslcmop["_id"]
346 nsr_id = nslcmop["nsInstanceId"]
347 task = asyncio.ensure_future(self.ns.scale(nsr_id, nslcmop_id))
348 self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "ns_scale", task)
349 return
350 elif command == "show":
tiernoc2564fe2019-01-28 16:18:56 +0000351 nsr_id = params
gcalvinoed7f6d42018-12-14 14:44:56 +0100352 try:
353 db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
354 print("nsr:\n _id={}\n operational-status: {}\n config-status: {}"
355 "\n detailed-status: {}\n deploy: {}\n tasks: {}"
356 "".format(nsr_id, db_nsr["operational-status"], db_nsr["config-status"],
357 db_nsr["detailed-status"],
358 db_nsr["_admin"]["deployed"], self.lcm_ns_tasks.get(nsr_id)))
359 except Exception as e:
360 print("nsr {} not found: {}".format(nsr_id, e))
361 sys.stdout.flush()
362 return
363 elif command == "deleted":
364 return # TODO cleaning of task just in case should be done
365 elif command in ("terminated", "instantiated", "scaled", "actioned"): # "scaled-cooldown-time"
366 return
367 elif topic == "nsi": # netslice LCM processes (instantiate, terminate, etc)
tierno307425f2020-01-26 23:35:59 +0000368 if command == "instantiate":
gcalvinoed7f6d42018-12-14 14:44:56 +0100369 # self.logger.debug("Instantiating Network Slice {}".format(nsilcmop["netsliceInstanceId"]))
370 nsilcmop = params
371 nsilcmop_id = nsilcmop["_id"] # slice operation id
372 nsir_id = nsilcmop["netsliceInstanceId"] # slice record id
373 task = asyncio.ensure_future(self.netslice.instantiate(nsir_id, nsilcmop_id))
374 self.lcm_tasks.register("nsi", nsir_id, nsilcmop_id, "nsi_instantiate", task)
375 return
tierno307425f2020-01-26 23:35:59 +0000376 elif command == "terminate":
gcalvinoed7f6d42018-12-14 14:44:56 +0100377 # self.logger.debug("Terminating Network Slice NS {}".format(nsilcmop["netsliceInstanceId"]))
378 nsilcmop = params
379 nsilcmop_id = nsilcmop["_id"] # slice operation id
380 nsir_id = nsilcmop["netsliceInstanceId"] # slice record id
381 self.lcm_tasks.cancel(topic, nsir_id)
382 task = asyncio.ensure_future(self.netslice.terminate(nsir_id, nsilcmop_id))
383 self.lcm_tasks.register("nsi", nsir_id, nsilcmop_id, "nsi_terminate", task)
384 return
385 elif command == "show":
tiernoc2564fe2019-01-28 16:18:56 +0000386 nsir_id = params
gcalvinoed7f6d42018-12-14 14:44:56 +0100387 try:
388 db_nsir = self.db.get_one("nsirs", {"_id": nsir_id})
389 print("nsir:\n _id={}\n operational-status: {}\n config-status: {}"
390 "\n detailed-status: {}\n deploy: {}\n tasks: {}"
391 "".format(nsir_id, db_nsir["operational-status"], db_nsir["config-status"],
392 db_nsir["detailed-status"],
393 db_nsir["_admin"]["deployed"], self.lcm_netslice_tasks.get(nsir_id)))
394 except Exception as e:
395 print("nsir {} not found: {}".format(nsir_id, e))
396 sys.stdout.flush()
397 return
398 elif command == "deleted":
399 return # TODO cleaning of task just in case should be done
400 elif command in ("terminated", "instantiated", "scaled", "actioned"): # "scaled-cooldown-time"
401 return
402 elif topic == "vim_account":
403 vim_id = params["_id"]
tiernof210c1c2019-10-16 09:09:58 +0000404 if command in ("create", "created"):
gcalvinoed7f6d42018-12-14 14:44:56 +0100405 task = asyncio.ensure_future(self.vim.create(params, order_id))
406 self.lcm_tasks.register("vim_account", vim_id, order_id, "vim_create", task)
407 return
calvinosanch9f9c6f22019-11-04 13:37:39 +0100408 elif command == "delete" or command == "deleted":
gcalvinoed7f6d42018-12-14 14:44:56 +0100409 self.lcm_tasks.cancel(topic, vim_id)
kuuse6a470c62019-07-10 13:52:45 +0200410 task = asyncio.ensure_future(self.vim.delete(params, order_id))
gcalvinoed7f6d42018-12-14 14:44:56 +0100411 self.lcm_tasks.register("vim_account", vim_id, order_id, "vim_delete", task)
412 return
413 elif command == "show":
414 print("not implemented show with vim_account")
415 sys.stdout.flush()
416 return
tiernof210c1c2019-10-16 09:09:58 +0000417 elif command in ("edit", "edited"):
gcalvinoed7f6d42018-12-14 14:44:56 +0100418 task = asyncio.ensure_future(self.vim.edit(params, order_id))
419 self.lcm_tasks.register("vim_account", vim_id, order_id, "vim_edit", task)
420 return
tiernof210c1c2019-10-16 09:09:58 +0000421 elif command == "deleted":
422 return # TODO cleaning of task just in case should be done
gcalvinoed7f6d42018-12-14 14:44:56 +0100423 elif topic == "wim_account":
424 wim_id = params["_id"]
tiernof210c1c2019-10-16 09:09:58 +0000425 if command in ("create", "created"):
gcalvinoed7f6d42018-12-14 14:44:56 +0100426 task = asyncio.ensure_future(self.wim.create(params, order_id))
427 self.lcm_tasks.register("wim_account", wim_id, order_id, "wim_create", task)
428 return
calvinosanch9f9c6f22019-11-04 13:37:39 +0100429 elif command == "delete" or command == "deleted":
gcalvinoed7f6d42018-12-14 14:44:56 +0100430 self.lcm_tasks.cancel(topic, wim_id)
kuuse6a470c62019-07-10 13:52:45 +0200431 task = asyncio.ensure_future(self.wim.delete(params, order_id))
gcalvinoed7f6d42018-12-14 14:44:56 +0100432 self.lcm_tasks.register("wim_account", wim_id, order_id, "wim_delete", task)
433 return
434 elif command == "show":
435 print("not implemented show with wim_account")
436 sys.stdout.flush()
437 return
tiernof210c1c2019-10-16 09:09:58 +0000438 elif command in ("edit", "edited"):
gcalvinoed7f6d42018-12-14 14:44:56 +0100439 task = asyncio.ensure_future(self.wim.edit(params, order_id))
440 self.lcm_tasks.register("wim_account", wim_id, order_id, "wim_edit", task)
441 return
tiernof210c1c2019-10-16 09:09:58 +0000442 elif command == "deleted":
443 return # TODO cleaning of task just in case should be done
gcalvinoed7f6d42018-12-14 14:44:56 +0100444 elif topic == "sdn":
445 _sdn_id = params["_id"]
tiernof210c1c2019-10-16 09:09:58 +0000446 if command in ("create", "created"):
gcalvinoed7f6d42018-12-14 14:44:56 +0100447 task = asyncio.ensure_future(self.sdn.create(params, order_id))
448 self.lcm_tasks.register("sdn", _sdn_id, order_id, "sdn_create", task)
449 return
calvinosanch9f9c6f22019-11-04 13:37:39 +0100450 elif command == "delete" or command == "deleted":
gcalvinoed7f6d42018-12-14 14:44:56 +0100451 self.lcm_tasks.cancel(topic, _sdn_id)
kuuse6a470c62019-07-10 13:52:45 +0200452 task = asyncio.ensure_future(self.sdn.delete(params, order_id))
gcalvinoed7f6d42018-12-14 14:44:56 +0100453 self.lcm_tasks.register("sdn", _sdn_id, order_id, "sdn_delete", task)
454 return
tiernof210c1c2019-10-16 09:09:58 +0000455 elif command in ("edit", "edited"):
gcalvinoed7f6d42018-12-14 14:44:56 +0100456 task = asyncio.ensure_future(self.sdn.edit(params, order_id))
457 self.lcm_tasks.register("sdn", _sdn_id, order_id, "sdn_edit", task)
458 return
tiernof210c1c2019-10-16 09:09:58 +0000459 elif command == "deleted":
460 return # TODO cleaning of task just in case should be done
gcalvinoed7f6d42018-12-14 14:44:56 +0100461 self.logger.critical("unknown topic {} and command '{}'".format(topic, command))
462
tiernoc0e42e22018-05-11 11:36:10 +0200463 async def kafka_read(self):
tierno16427352019-04-22 11:37:36 +0000464 self.logger.debug("Task kafka_read Enter with worker_id={}".format(self.worker_id))
tiernoc0e42e22018-05-11 11:36:10 +0200465 # future = asyncio.Future()
gcalvinoed7f6d42018-12-14 14:44:56 +0100466 self.consecutive_errors = 0
467 self.first_start = True
468 while self.consecutive_errors < 10:
tiernoc0e42e22018-05-11 11:36:10 +0200469 try:
magnussonle9198bb2020-01-21 13:00:51 +0100470 topics = ("ns", "vim_account", "wim_account", "sdn", "nsi", "k8scluster", "k8srepo", "pla")
tierno16427352019-04-22 11:37:36 +0000471 topics_admin = ("admin", )
472 await asyncio.gather(
473 self.msg.aioread(topics, self.loop, self.kafka_read_callback),
474 self.msg_admin.aioread(topics_admin, self.loop, self.kafka_read_callback, group_id=False)
475 )
tiernoc0e42e22018-05-11 11:36:10 +0200476
gcalvinoed7f6d42018-12-14 14:44:56 +0100477 except LcmExceptionExit:
478 self.logger.debug("Bye!")
479 break
tiernoc0e42e22018-05-11 11:36:10 +0200480 except Exception as e:
481 # if not first_start is the first time after starting. So leave more time and wait
482 # to allow kafka starts
gcalvinoed7f6d42018-12-14 14:44:56 +0100483 if self.consecutive_errors == 8 if not self.first_start else 30:
tiernoc0e42e22018-05-11 11:36:10 +0200484 self.logger.error("Task kafka_read task exit error too many errors. Exception: {}".format(e))
485 raise
gcalvinoed7f6d42018-12-14 14:44:56 +0100486 self.consecutive_errors += 1
tiernoc0e42e22018-05-11 11:36:10 +0200487 self.logger.error("Task kafka_read retrying after Exception {}".format(e))
gcalvinoed7f6d42018-12-14 14:44:56 +0100488 wait_time = 2 if not self.first_start else 5
tiernoc0e42e22018-05-11 11:36:10 +0200489 await asyncio.sleep(wait_time, loop=self.loop)
490
491 # self.logger.debug("Task kafka_read terminating")
492 self.logger.debug("Task kafka_read exit")
493
494 def start(self):
tierno22f4f9c2018-06-11 18:53:39 +0200495
496 # check RO version
497 self.loop.run_until_complete(self.check_RO_version())
498
tiernoc0e42e22018-05-11 11:36:10 +0200499 self.loop.run_until_complete(asyncio.gather(
500 self.kafka_read(),
501 self.kafka_ping()
502 ))
503 # TODO
504 # self.logger.debug("Terminating cancelling creation tasks")
tiernoca2e16a2018-06-29 15:25:24 +0200505 # self.lcm_tasks.cancel("ALL", "create")
tiernoc0e42e22018-05-11 11:36:10 +0200506 # timeout = 200
507 # while self.is_pending_tasks():
508 # self.logger.debug("Task kafka_read terminating. Waiting for tasks termination")
509 # await asyncio.sleep(2, loop=self.loop)
510 # timeout -= 2
511 # if not timeout:
tiernoca2e16a2018-06-29 15:25:24 +0200512 # self.lcm_tasks.cancel("ALL", "ALL")
tiernoc0e42e22018-05-11 11:36:10 +0200513 self.loop.close()
514 self.loop = None
515 if self.db:
516 self.db.db_disconnect()
517 if self.msg:
518 self.msg.disconnect()
tierno16427352019-04-22 11:37:36 +0000519 if self.msg_admin:
520 self.msg_admin.disconnect()
tiernoc0e42e22018-05-11 11:36:10 +0200521 if self.fs:
522 self.fs.fs_disconnect()
523
tiernoc0e42e22018-05-11 11:36:10 +0200524 def read_config_file(self, config_file):
525 # TODO make a [ini] + yaml inside parser
526 # the configparser library is not suitable, because it does not admit comments at the end of line,
527 # and not parse integer or boolean
528 try:
tierno744303e2020-01-13 16:46:31 +0000529 # read file as yaml format
tiernoc0e42e22018-05-11 11:36:10 +0200530 with open(config_file) as f:
tiernoda6fb102019-11-23 00:36:52 +0000531 conf = yaml.load(f, Loader=yaml.Loader)
tierno744303e2020-01-13 16:46:31 +0000532 # Ensure all sections are not empty
533 for k in ("global", "timeout", "RO", "VCA", "database", "storage", "message"):
534 if not conf.get(k):
535 conf[k] = {}
536
537 # read all environ that starts with OSMLCM_
tiernoc0e42e22018-05-11 11:36:10 +0200538 for k, v in environ.items():
539 if not k.startswith("OSMLCM_"):
540 continue
tierno744303e2020-01-13 16:46:31 +0000541 subject, _, item = k[7:].lower().partition("_")
542 if not item:
tierno17a612f2018-10-23 11:30:42 +0200543 continue
tierno744303e2020-01-13 16:46:31 +0000544 if subject in ("ro", "vca"):
tierno17a612f2018-10-23 11:30:42 +0200545 # put in capital letter
tierno744303e2020-01-13 16:46:31 +0000546 subject = subject.upper()
tiernoc0e42e22018-05-11 11:36:10 +0200547 try:
tierno744303e2020-01-13 16:46:31 +0000548 if item == "port" or subject == "timeout":
549 conf[subject][item] = int(v)
tiernoc0e42e22018-05-11 11:36:10 +0200550 else:
tierno744303e2020-01-13 16:46:31 +0000551 conf[subject][item] = v
tiernoc0e42e22018-05-11 11:36:10 +0200552 except Exception as e:
tierno744303e2020-01-13 16:46:31 +0000553 self.logger.warning("skipping environ '{}' on exception '{}'".format(k, e))
554
555 # backward compatibility of VCA parameters
556
557 if 'pubkey' in conf["VCA"]:
558 conf["VCA"]['public_key'] = conf["VCA"].pop('pubkey')
559 if 'cacert' in conf["VCA"]:
560 conf["VCA"]['ca_cert'] = conf["VCA"].pop('cacert')
561 if 'apiproxy' in conf["VCA"]:
562 conf["VCA"]['api_proxy'] = conf["VCA"].pop('apiproxy')
563
564 if 'enableosupgrade' in conf["VCA"]:
565 conf["VCA"]['enable_os_upgrade'] = conf["VCA"].pop('enableosupgrade')
566 if isinstance(conf["VCA"].get('enable_os_upgrade'), str):
567 if conf["VCA"]['enable_os_upgrade'].lower() == 'false':
568 conf["VCA"]['enable_os_upgrade'] = False
569 elif conf["VCA"]['enable_os_upgrade'].lower() == 'true':
570 conf["VCA"]['enable_os_upgrade'] = True
571
572 if 'aptmirror' in conf["VCA"]:
573 conf["VCA"]['apt_mirror'] = conf["VCA"].pop('aptmirror')
tiernoc0e42e22018-05-11 11:36:10 +0200574
575 return conf
576 except Exception as e:
577 self.logger.critical("At config file '{}': {}".format(config_file, e))
578 exit(1)
579
tierno16427352019-04-22 11:37:36 +0000580 @staticmethod
581 def get_process_id():
582 """
583 Obtain a unique ID for this process. If running from inside docker, it will get docker ID. If not it
584 will provide a random one
585 :return: Obtained ID
586 """
587 # Try getting docker id. If fails, get pid
588 try:
589 with open("/proc/self/cgroup", "r") as f:
590 text_id_ = f.readline()
591 _, _, text_id = text_id_.rpartition("/")
592 text_id = text_id.replace('\n', '')[:12]
593 if text_id:
594 return text_id
595 except Exception:
596 pass
597 # Return a random id
598 return ''.join(random_choice("0123456789abcdef") for _ in range(12))
599
tiernoc0e42e22018-05-11 11:36:10 +0200600
tierno275411e2018-05-16 14:33:32 +0200601def usage():
602 print("""Usage: {} [options]
quilesj7e13aeb2019-10-08 13:34:55 +0200603 -c|--config [configuration_file]: loads the configuration file (default: ./lcm.cfg)
tiernoa9843d82018-10-24 10:44:20 +0200604 --health-check: do not run lcm, but inspect kafka bus to determine if lcm is healthy
tierno275411e2018-05-16 14:33:32 +0200605 -h|--help: shows this help
606 """.format(sys.argv[0]))
tierno750b2452018-05-17 16:39:29 +0200607 # --log-socket-host HOST: send logs to this host")
608 # --log-socket-port PORT: send logs using this port (default: 9022)")
tierno275411e2018-05-16 14:33:32 +0200609
610
tiernoc0e42e22018-05-11 11:36:10 +0200611if __name__ == '__main__':
quilesj7e13aeb2019-10-08 13:34:55 +0200612
tierno275411e2018-05-16 14:33:32 +0200613 try:
tierno8c16b052020-02-05 15:08:32 +0000614 # print("SYS.PATH='{}'".format(sys.path))
tierno275411e2018-05-16 14:33:32 +0200615 # load parameters and configuration
quilesj7e13aeb2019-10-08 13:34:55 +0200616 # -h
617 # -c value
618 # --config value
619 # --help
620 # --health-check
tiernoa9843d82018-10-24 10:44:20 +0200621 opts, args = getopt.getopt(sys.argv[1:], "hc:", ["config=", "help", "health-check"])
tierno275411e2018-05-16 14:33:32 +0200622 # TODO add "log-socket-host=", "log-socket-port=", "log-file="
623 config_file = None
624 for o, a in opts:
625 if o in ("-h", "--help"):
626 usage()
627 sys.exit()
628 elif o in ("-c", "--config"):
629 config_file = a
tiernoa9843d82018-10-24 10:44:20 +0200630 elif o == "--health-check":
tierno94f06112020-02-11 12:38:19 +0000631 from osm_lcm.lcm_hc import health_check
632 health_check(health_check_file, Lcm.ping_interval_pace)
tierno275411e2018-05-16 14:33:32 +0200633 # elif o == "--log-socket-port":
634 # log_socket_port = a
635 # elif o == "--log-socket-host":
636 # log_socket_host = a
637 # elif o == "--log-file":
638 # log_file = a
639 else:
640 assert False, "Unhandled option"
quilesj7e13aeb2019-10-08 13:34:55 +0200641
tierno275411e2018-05-16 14:33:32 +0200642 if config_file:
643 if not path.isfile(config_file):
quilesj7e13aeb2019-10-08 13:34:55 +0200644 print("configuration file '{}' does not exist".format(config_file), file=sys.stderr)
tierno275411e2018-05-16 14:33:32 +0200645 exit(1)
646 else:
647 for config_file in (__file__[:__file__.rfind(".")] + ".cfg", "./lcm.cfg", "/etc/osm/lcm.cfg"):
648 if path.isfile(config_file):
649 break
650 else:
tierno17a612f2018-10-23 11:30:42 +0200651 print("No configuration file 'lcm.cfg' found neither at local folder nor at /etc/osm/", file=sys.stderr)
tierno275411e2018-05-16 14:33:32 +0200652 exit(1)
653 lcm = Lcm(config_file)
tierno3e359b12019-02-03 02:29:13 +0100654 lcm.start()
tierno22f4f9c2018-06-11 18:53:39 +0200655 except (LcmException, getopt.GetoptError) as e:
tierno275411e2018-05-16 14:33:32 +0200656 print(str(e), file=sys.stderr)
657 # usage()
658 exit(1)