adding flake8 test
[osm/LCM.git] / osm_lcm / lcm.py
index 69daaa1..a4395eb 100644 (file)
@@ -4,29 +4,31 @@
 import asyncio
 import yaml
 import ROclient
+import logging
+import logging.handlers
+import getopt
+import functools
+import sys
 from osm_common import dbmemory
 from osm_common import dbmongo
 from osm_common import fslocal
 from osm_common import msglocal
 from osm_common import msgkafka
-import logging
-import functools
-import sys
 from osm_common.dbbase import DbException
 from osm_common.fsbase import FsException
 from osm_common.msgbase import MsgException
-from os import environ
-# from vca import DeployApplication, RemoveApplication
+from os import environ, path
 from n2vc.vnf import N2VC
 from n2vc import version as N2VC_version
-# import os.path
-# import time
 
 from copy import deepcopy
 from http import HTTPStatus
 from time import time
 
 
+__author__ = "Alfonso Tierno"
+
+
 class LcmException(Exception):
     pass
 
@@ -54,9 +56,9 @@ class Lcm:
         # load configuration
         config = self.read_config_file(config_file)
         self.config = config
-        self.ro_config={
+        self.ro_config = {
             "endpoint_url": "http://{}:{}/openmano".format(config["RO"]["host"], config["RO"]["port"]),
-            "tenant":  config.get("tenant", "osm"),
+            "tenant": config.get("tenant", "osm"),
             "logger_name": "lcm.ROclient",
             "loglevel": "ERROR",
         }
@@ -188,11 +190,11 @@ class Lcm:
 
             step = "Attach vim to RO tenant"
             vim_RO = {"vim_tenant_name": vim_content["vim_tenant_name"],
-                "vim_username": vim_content["vim_user"],
-                "vim_password": vim_content["vim_password"],
-                "config": vim_content["config"]
-            }
-            desc = await RO.attach_datacenter(RO_vim_id , descriptor=vim_RO)
+                      "vim_username": vim_content["vim_user"],
+                      "vim_password": vim_content["vim_password"],
+                      "config": vim_content["config"]
+                      }
+            desc = await RO.attach_datacenter(RO_vim_id, descriptor=vim_RO)
             db_vim["_admin"]["operationalState"] = "ENABLED"
             self.update_db("vim_accounts", vim_id, db_vim)
 
@@ -208,7 +210,7 @@ class Lcm:
         finally:
             if exc and db_vim:
                 db_vim["_admin"]["operationalState"] = "ERROR"
-                db_vim["_admin"]["detailed-status"] = "ERROR {}: {}".format(step , exc)
+                db_vim["_admin"]["detailed-status"] = "ERROR {}: {}".format(step, exc)
                 self.update_db("vim_accounts", vim_id, db_vim)
 
     async def vim_edit(self, vim_content, order_id):
@@ -234,7 +236,7 @@ class Lcm:
                 vim_RO.pop("vim_user", None)
                 vim_RO.pop("vim_password", None)
                 if vim_RO:
-                    desc = await RO.edit("vim", RO_vim_id, descriptor=vim_RO)
+                    await RO.edit("vim", RO_vim_id, descriptor=vim_RO)
 
                 step = "Editing vim-account at RO tenant"
                 vim_RO = {}
@@ -244,7 +246,7 @@ class Lcm:
                 if "vim_user" in vim_content:
                     vim_content["vim_username"] = vim_content["vim_user"]
                 if vim_RO:
-                    desc = await RO.edit("vim_account", RO_vim_id, descriptor=vim_RO)
+                    await RO.edit("vim_account", RO_vim_id, descriptor=vim_RO)
                 db_vim["_admin"]["operationalState"] = "ENABLED"
                 self.update_db("vim_accounts", vim_id, db_vim)
 
@@ -260,7 +262,7 @@ class Lcm:
         finally:
             if exc and db_vim:
                 db_vim["_admin"]["operationalState"] = "ERROR"
-                db_vim["_admin"]["detailed-status"] = "ERROR {}: {}".format(step , exc)
+                db_vim["_admin"]["detailed-status"] = "ERROR {}: {}".format(step, exc)
                 self.update_db("vim_accounts", vim_id, db_vim)
 
     async def vim_delete(self, vim_id, order_id):
@@ -307,7 +309,7 @@ class Lcm:
         finally:
             if exc and db_vim:
                 db_vim["_admin"]["operationalState"] = "ERROR"
-                db_vim["_admin"]["detailed-status"] = "ERROR {}: {}".format(step , exc)
+                db_vim["_admin"]["detailed-status"] = "ERROR {}: {}".format(step, exc)
                 self.update_db("vim_accounts", vim_id, db_vim)
 
     async def sdn_create(self, sdn_content, order_id):
@@ -323,7 +325,7 @@ class Lcm:
                 db_sdn["_admin"] = {}
             if "deployed" not in db_sdn["_admin"]:
                 db_sdn["_admin"]["deployed"] = {}
-            db_sdn["_admin"]["deployed"]["RO"] =  None
+            db_sdn["_admin"]["deployed"]["RO"] = None
 
             step = "Creating sdn at RO"
             RO = ROclient.ROClient(self.loop, **self.ro_config)
@@ -350,7 +352,7 @@ class Lcm:
         finally:
             if exc and db_sdn:
                 db_sdn["_admin"]["operationalState"] = "ERROR"
-                db_sdn["_admin"]["detailed-status"] = "ERROR {}: {}".format(step , exc)
+                db_sdn["_admin"]["detailed-status"] = "ERROR {}: {}".format(step, exc)
                 self.update_db("sdns", sdn_id, db_sdn)
 
     async def sdn_edit(self, sdn_content, order_id):
@@ -373,7 +375,7 @@ class Lcm:
                 sdn_RO.pop("schema_type", None)
                 sdn_RO.pop("description", None)
                 if sdn_RO:
-                    desc = await RO.edit("sdn", RO_sdn_id, descriptor=sdn_RO)
+                    await RO.edit("sdn", RO_sdn_id, descriptor=sdn_RO)
                 db_sdn["_admin"]["operationalState"] = "ENABLED"
                 self.update_db("sdns", sdn_id, db_sdn)
 
@@ -389,7 +391,7 @@ class Lcm:
         finally:
             if exc and db_sdn:
                 db_sdn["_admin"]["operationalState"] = "ERROR"
-                db_sdn["_admin"]["detailed-status"] = "ERROR {}: {}".format(step , exc)
+                db_sdn["_admin"]["detailed-status"] = "ERROR {}: {}".format(step, exc)
                 self.update_db("sdns", sdn_id, db_sdn)
 
     async def sdn_delete(self, sdn_id, order_id):
@@ -427,7 +429,7 @@ class Lcm:
         finally:
             if exc and db_sdn:
                 db_sdn["_admin"]["operationalState"] = "ERROR"
-                db_sdn["_admin"]["detailed-status"] = "ERROR {}: {}".format(step , exc)
+                db_sdn["_admin"]["detailed-status"] = "ERROR {}: {}".format(step, exc)
                 self.update_db("sdns", sdn_id, db_sdn)
 
     def vnfd2RO(self, vnfd, new_id=None):
@@ -453,7 +455,8 @@ class Lcm:
                         vdu["cloud-init-file"]
                     )
                     ci_file = self.fs.file_open(clout_init_file, "r")
-                    # TODO: detect if binary or text. Propose to read as binary and try to decode to utf8. If fails convert to base 64 or similar
+                    # TODO: detect if binary or text. Propose to read as binary and try to decode to utf8. If fails
+                    #  convert to base 64 or similar
                     clout_init_content = ci_file.read()
                     ci_file.close()
                     ci_file = None
@@ -466,7 +469,8 @@ class Lcm:
             if ci_file:
                 ci_file.close()
 
-    def n2vc_callback(self, model_name, application_name, status, message, db_nsr, db_nslcmop, vnf_member_index, task=None):
+    def n2vc_callback(self, model_name, application_name, status, message, db_nsr, db_nslcmop, member_vnf_index,
+                      task=None):
         """
         Callback both for charm status change and task completion
         :param model_name: Charm model name
@@ -483,7 +487,7 @@ class Lcm:
         :param message: detailed message error
         :param db_nsr: nsr database content
         :param db_nslcmop: nslcmop database content
-        :param vnf_member_index: NSD vnf-member-index
+        :param member_vnf_index: NSD member-vnf-index
         :param task: None for charm status change, or task for completion task callback
         :return:
         """
@@ -496,7 +500,7 @@ class Lcm:
             nsr_lcm = db_nsr["_admin"]["deployed"]
             ns_action = db_nslcmop["lcmOperationType"]
             logging_text = "Task ns={} {}={} [n2vc_callback] vnf_index={}".format(nsr_id, ns_action, nslcmop_id,
-                                                                                  vnf_member_index)
+                                                                                  member_vnf_index)
 
             if task:
                 if task.cancelled():
@@ -509,8 +513,8 @@ class Lcm:
                     if exc:
                         self.logger.error(logging_text + " task Exception={}".format(exc))
                         if ns_action in ("instantiate", "terminate"):
-                            nsr_lcm["VCA"][vnf_member_index]['operational-status'] = "error"
-                            nsr_lcm["VCA"][vnf_member_index]['detailed-status'] = str(exc)
+                            nsr_lcm["VCA"][member_vnf_index]['operational-status'] = "error"
+                            nsr_lcm["VCA"][member_vnf_index]['detailed-status'] = str(exc)
                         elif ns_action == "action":
                             db_nslcmop["operationState"] = "FAILED"
                             db_nslcmop["detailed-status"] = str(exc)
@@ -530,10 +534,10 @@ class Lcm:
                         return
             elif status:
                 self.logger.debug(logging_text + " Enter status={}".format(status))
-                if nsr_lcm["VCA"][vnf_member_index]['operational-status'] == status:
+                if nsr_lcm["VCA"][member_vnf_index]['operational-status'] == status:
                     return  # same status, ignore
-                nsr_lcm["VCA"][vnf_member_index]['operational-status'] = status
-                nsr_lcm["VCA"][vnf_member_index]['detailed-status'] = str(message)
+                nsr_lcm["VCA"][member_vnf_index]['operational-status'] = status
+                nsr_lcm["VCA"][member_vnf_index]['detailed-status'] = str(message)
             else:
                 self.logger.critical(logging_text + " Enter with bad parameters", exc_info=True)
                 return
@@ -551,11 +555,12 @@ class Lcm:
                 if vca_status != "active":
                     all_active = False
                 elif vca_status in ("error", "blocked"):
-                    n2vc_error_text.append("member_vnf_index={} {}: {}".format(vnf_member_index, vca_status,
-                                                                           vca_info["detailed-status"]))
+                    n2vc_error_text.append("member_vnf_index={} {}: {}".format(member_vnf_index, vca_status,
+                                                                               vca_info["detailed-status"]))
 
             if all_active:
-                self.logger.debug("[n2vc_callback] ns_instantiate={} vnf_index={} All active".format(nsr_id, vnf_member_index))
+                self.logger.debug("[n2vc_callback] ns_instantiate={} vnf_index={} All active".format(nsr_id,
+                                                                                                     member_vnf_index))
                 db_nsr["config-status"] = "configured"
                 db_nsr["detailed-status"] = "done"
                 db_nslcmop["operationState"] = "COMPLETED"
@@ -580,7 +585,7 @@ class Lcm:
             update_nsr = update_nslcmop = True
 
         except Exception as e:
-            self.logger.critical("[n2vc_callback] vnf_index={} Exception {}".format(vnf_member_index, e), exc_info=True)
+            self.logger.critical("[n2vc_callback] vnf_index={} Exception {}".format(member_vnf_index, e), exc_info=True)
         finally:
             try:
                 if update_nslcmop:
@@ -589,7 +594,7 @@ class Lcm:
                     self.update_db("nsrs", nsr_id, db_nsr)
             except Exception as e:
                 self.logger.critical("[n2vc_callback] vnf_index={} Update database Exception {}".format(
-                    vnf_member_index, e), exc_info=True)
+                    member_vnf_index, e), exc_info=True)
 
     def ns_params_2_RO(self, ns_params):
         """
@@ -598,6 +603,7 @@ class Lcm:
         :return: The RO ns descriptor
         """
         vim_2_RO = {}
+
         def vim_account_2_RO(vim_account):
             if vim_account in vim_2_RO:
                 return vim_2_RO[vim_account]
@@ -643,8 +649,8 @@ class Lcm:
                                 "netmap-use": vim_net,
                                 "datacenter": vim_account_2_RO(vim_account)
                             })
-                    else:  #isinstance str
-                        RO_vld["sites"].append({"netmap-use":  vld["vim-network-name"]})
+                    else:  # isinstance str
+                        RO_vld["sites"].append({"netmap-use": vld["vim-network-name"]})
                 if RO_vld:
                     RO_ns_params["networks"][vld["name"]] = RO_vld
         return RO_ns_params
@@ -777,7 +783,7 @@ class Lcm:
             step = ns_status_detailed = "Waiting ns ready at RO"
             db_nsr["detailed-status"] = ns_status_detailed
             self.logger.debug(logging_text + step + " RO_ns_id={}".format(RO_nsr_id))
-            deployment_timeout = 2*3600   # Two hours
+            deployment_timeout = 2 * 3600   # Two hours
             while deployment_timeout > 0:
                 desc = await RO.show("ns", RO_nsr_id)
                 ns_status, ns_status_info = RO.check_ns_status(desc)
@@ -785,15 +791,20 @@ class Lcm:
                 if ns_status == "ERROR":
                     raise ROclient.ROClientException(ns_status_info)
                 elif ns_status == "BUILD":
+                    db_nsr_detailed_status_old = db_nsr["detailed-status"]
                     db_nsr["detailed-status"] = ns_status_detailed + "; {}".format(ns_status_info)
-                    self.update_db("nsrs", nsr_id, db_nsr)
+                    if db_nsr_detailed_status_old != db_nsr["detailed-status"]:
+                        self.update_db("nsrs", nsr_id, db_nsr)
                 elif ns_status == "ACTIVE":
-                    step = "Getting ns VIM information"
-                    ns_RO_info = nsr_lcm["nsr_ip"] = RO.get_ns_vnf_info(desc)
-                    break
+                    step = "Waiting for management IP address from VIM"
+                    try:
+                        ns_RO_info = nsr_lcm["nsr_ip"] = RO.get_ns_vnf_info(desc)
+                        break
+                    except ROclient.ROClientException as e:
+                        if e.http_code != 409:  # IP address is not ready return code is 409 CONFLICT
+                            raise e
                 else:
                     assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
-
                 await asyncio.sleep(5, loop=self.loop)
                 deployment_timeout -= 5
             if deployment_timeout <= 0:
@@ -827,7 +838,7 @@ class Lcm:
                 #     yield from asyncio.wait_for(task, 30.0)
                 #     self.logger.debug("Logged into N2VC!")
 
-                ## await self.n2vc.login()
+                # # await self.n2vc.login()
 
                 # Note: The charm needs to exist on disk at the location
                 # specified by charm_path.
@@ -860,7 +871,8 @@ class Lcm:
                     "vnfd_id": vnfd_id,
                 }
 
-                self.logger.debug("Task create_ns={} Passing artifacts path '{}' for {}".format(nsr_id, charm_path, proxy_charm))
+                self.logger.debug("Task create_ns={} Passing artifacts path '{}' for {}".format(nsr_id, charm_path,
+                                                                                                proxy_charm))
                 task = asyncio.ensure_future(
                     self.n2vc.DeployCharms(
                         model_name,          # The network service name
@@ -1015,7 +1027,7 @@ class Lcm:
                 try:
                     step = db_nsr["detailed-status"] = "Deleting ns at RO"
                     self.logger.debug(logging_text + step)
-                    desc = await RO.delete("ns", RO_nsr_id)
+                    await RO.delete("ns", RO_nsr_id)
                     nsr_lcm["RO"]["nsr_id"] = None
                     nsr_lcm["RO"]["nsr_status"] = "DELETED"
                 except ROclient.ROClientException as e:
@@ -1023,7 +1035,7 @@ class Lcm:
                         nsr_lcm["RO"]["nsr_id"] = None
                         nsr_lcm["RO"]["nsr_status"] = "DELETED"
                         self.logger.debug(logging_text + "RO_ns_id={} already deleted".format(RO_nsr_id))
-                    elif e.http_code == 409:   #conflict
+                    elif e.http_code == 409:   # conflict
                         failed_detail.append("RO_ns_id={} delete conflict: {}".format(RO_nsr_id, e))
                         self.logger.debug(logging_text + failed_detail[-1])
                     else:
@@ -1035,14 +1047,14 @@ class Lcm:
             if RO_nsd_id:
                 try:
                     step = db_nsr["detailed-status"] = "Deleting nsd at RO"
-                    desc = await RO.delete("nsd", RO_nsd_id)
+                    await RO.delete("nsd", RO_nsd_id)
                     self.logger.debug(logging_text + "RO_nsd_id={} deleted".format(RO_nsd_id))
                     nsr_lcm["RO"]["nsd_id"] = None
                 except ROclient.ROClientException as e:
                     if e.http_code == 404:  # not found
                         nsr_lcm["RO"]["nsd_id"] = None
                         self.logger.debug(logging_text + "RO_nsd_id={} already deleted".format(RO_nsd_id))
-                    elif e.http_code == 409:   #conflict
+                    elif e.http_code == 409:   # conflict
                         failed_detail.append("RO_nsd_id={} delete conflict: {}".format(RO_nsd_id, e))
                         self.logger.debug(logging_text + failed_detail[-1])
                     else:
@@ -1054,14 +1066,14 @@ class Lcm:
                     continue
                 try:
                     step = db_nsr["detailed-status"] = "Deleting vnfd={} at RO".format(vnf_id)
-                    desc = await RO.delete("vnfd", RO_vnfd_id)
+                    await RO.delete("vnfd", RO_vnfd_id)
                     self.logger.debug(logging_text + "RO_vnfd_id={} deleted".format(RO_vnfd_id))
                     nsr_lcm["RO"]["vnfd_id"][vnf_id] = None
                 except ROclient.ROClientException as e:
                     if e.http_code == 404:  # not found
                         nsr_lcm["RO"]["vnfd_id"][vnf_id] = None
                         self.logger.debug(logging_text + "RO_vnfd_id={} already deleted ".format(RO_vnfd_id))
-                    elif e.http_code == 409:   #conflict
+                    elif e.http_code == 409:   # conflict
                         failed_detail.append("RO_vnfd_id={} delete conflict: {}".format(RO_vnfd_id, e))
                         self.logger.debug(logging_text + failed_detail[-1])
                     else:
@@ -1144,18 +1156,18 @@ class Lcm:
             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
             nsr_lcm = db_nsr["_admin"].get("deployed")
-            vnf_index = db_nslcmop["operationParams"]["vnf_member_index"]
+            vnf_index = db_nslcmop["operationParams"]["member_vnf_index"]
 
-            #TODO check if ns is in a proper status
+            # TODO check if ns is in a proper status
             vca_deployed = nsr_lcm["VCA"].get(vnf_index)
             if not vca_deployed:
-                raise LcmException("charm for vnf_member_index={} is not deployed".format(vnf_index))
+                raise LcmException("charm for member_vnf_index={} is not deployed".format(vnf_index))
             model_name = vca_deployed.get("model")
             application_name = vca_deployed.get("application")
             if not model_name or not application_name:
-                raise LcmException("charm for vnf_member_index={} is not properly deployed".format(vnf_index))
+                raise LcmException("charm for member_vnf_index={} is not properly deployed".format(vnf_index))
             if vca_deployed["operational-status"] != "active":
-                raise LcmException("charm for vnf_member_index={} operational_status={} not 'active'".format(
+                raise LcmException("charm for member_vnf_index={} operational_status={} not 'active'".format(
                     vnf_index, vca_deployed["operational-status"]))
             primitive = db_nslcmop["operationParams"]["primitive"]
             primitive_params = db_nslcmop["operationParams"]["primitive_params"]
@@ -1231,9 +1243,9 @@ class Lcm:
         """
         if topic == "ns":
             lcm_tasks = self.lcm_ns_tasks
-        elif topic== "vim_account":
+        elif topic == "vim_account":
             lcm_tasks = self.lcm_vim_tasks
-        elif topic== "sdn":
+        elif topic == "sdn":
             lcm_tasks = self.lcm_sdn_tasks
 
         if not lcm_tasks.get(_id):
@@ -1253,7 +1265,7 @@ class Lcm:
         self.pings_not_received = 1
         while True:
             try:
-                await self.msg.aiowrite("admin", "ping", {"from": "lcm", "to": "lcm"},  self.loop)
+                await self.msg.aiowrite("admin", "ping", {"from": "lcm", "to": "lcm"}, self.loop)
                 # time between pings are low when it is not received and at starting
                 wait_time = 5 if not kafka_has_received else 120
                 if not self.pings_not_received:
@@ -1344,12 +1356,11 @@ class Lcm:
                     elif command == "show":
                         try:
                             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
-                            print(
-                            "nsr:\n    _id={}\n    operational-status: {}\n    config-status: {}\n    detailed-status: "
-                            "{}\n    deploy: {}\n    tasks: {}".format(
-                                nsr_id, db_nsr["operational-status"],
-                                db_nsr["config-status"], db_nsr["detailed-status"],
-                                db_nsr["_admin"]["deployed"], self.lcm_ns_tasks.get(nsr_id)))
+                            print("nsr:\n    _id={}\n    operational-status: {}\n    config-status: {}"
+                                  "\n    detailed-status: {}\n    deploy: {}\n    tasks: {}"
+                                  "".format(nsr_id, db_nsr["operational-status"], db_nsr["config-status"],
+                                            db_nsr["detailed-status"],
+                                            db_nsr["_admin"]["deployed"], self.lcm_ns_tasks.get(nsr_id)))
                         except Exception as e:
                             print("nsr {} not found: {}".format(nsr_id, e))
                         sys.stdout.flush()
@@ -1442,7 +1453,6 @@ class Lcm:
         if self.fs:
             self.fs.fs_disconnect()
 
-
     def read_config_file(self, config_file):
         # TODO make a [ini] + yaml inside parser
         # the configparser library is not suitable, because it does not admit comments at the end of line,
@@ -1474,9 +1484,49 @@ class Lcm:
             exit(1)
 
 
-if __name__ == '__main__':
+def usage():
+    print("""Usage: {} [options]
+        -c|--config [configuration_file]: loads the configuration file (default: ./nbi.cfg)
+        -h|--help: shows this help
+        """.format(sys.argv[0]))
+    # --log-socket-host HOST: send logs to this host")
+    # --log-socket-port PORT: send logs using this port (default: 9022)")
 
-    config_file = "lcm.cfg"
-    lcm = Lcm(config_file)
 
-    lcm.start()
+if __name__ == '__main__':
+    try:
+        # load parameters and configuration
+        opts, args = getopt.getopt(sys.argv[1:], "hc:", ["config=", "help"])
+        # TODO add  "log-socket-host=", "log-socket-port=", "log-file="
+        config_file = None
+        for o, a in opts:
+            if o in ("-h", "--help"):
+                usage()
+                sys.exit()
+            elif o in ("-c", "--config"):
+                config_file = a
+            # elif o == "--log-socket-port":
+            #     log_socket_port = a
+            # elif o == "--log-socket-host":
+            #     log_socket_host = a
+            # elif o == "--log-file":
+            #     log_file = a
+            else:
+                assert False, "Unhandled option"
+        if config_file:
+            if not path.isfile(config_file):
+                print("configuration file '{}' that not exist".format(config_file), file=sys.stderr)
+                exit(1)
+        else:
+            for config_file in (__file__[:__file__.rfind(".")] + ".cfg", "./lcm.cfg", "/etc/osm/lcm.cfg"):
+                if path.isfile(config_file):
+                    break
+            else:
+                print("No configuration file 'nbi.cfg' found neither at local folder nor at /etc/osm/", file=sys.stderr)
+                exit(1)
+        lcm = Lcm(config_file)
+        lcm.start()
+    except getopt.GetoptError as e:
+        print(str(e), file=sys.stderr)
+        # usage()
+        exit(1)