Fix bug 1721: charms can include reference to .charm files
[osm/osmclient.git] / osmclient / common / package_tool.py
index e3dbd1a..436053c 100644 (file)
@@ -54,6 +54,7 @@ class PackageTool(object):
         detailed,
         netslice_subnets,
         netslice_vlds,
+        old,
     ):
         """
         **Create a package descriptor**
@@ -71,6 +72,7 @@ class PackageTool(object):
             - detailed: include all possible values for NSD, VNFD, NST
             - netslice_subnets: number of netslice_subnets for the NST
             - netslice_vlds: number of virtual link descriptors for the NST
+            - old: flag to create a descriptor using the previous OSM format (pre SOL006, OSM<9)
 
         :return: status
         """
@@ -79,7 +81,7 @@ class PackageTool(object):
         file_loader = PackageLoader("osmclient")
         env = Environment(loader=file_loader)
         if package_type == "ns":
-            template = env.get_template("nsd.yaml.j2")
+            template = env.get_template("nsd.yaml.j2" if not old else "nsd_old.yaml.j2")
             content = {
                 "name": package_name,
                 "vendor": vendor,
@@ -89,7 +91,9 @@ class PackageTool(object):
                 "detailed": detailed,
             }
         elif package_type == "vnf":
-            template = env.get_template("vnfd.yaml.j2")
+            template = env.get_template(
+                "vnfd.yaml.j2" if not old else "vnfd_old.yaml.j2"
+            )
             content = {
                 "name": package_name,
                 "vendor": vendor,
@@ -103,6 +107,7 @@ class PackageTool(object):
                 "detailed": detailed,
             }
         elif package_type == "nst":
+            # TODO: repo-index did not support nst in OSM<9, no changes in template
             template = env.get_template("nst.yaml.j2")
             content = {
                 "name": package_name,
@@ -117,9 +122,9 @@ class PackageTool(object):
                 "Wrong descriptor type {}. Options: ns, vnf, nst".format(package_type)
             )
 
-        # print("To be rendered: {}".format(content))
+        self._logger.debug("To be rendered: {}".format(content))
         output = template.render(content)
-        # print(output)
+        self._logger.debug(output)
 
         structure = self.discover_folder_structure(
             base_directory, package_name, override
@@ -149,27 +154,31 @@ class PackageTool(object):
             descriptors_paths = [
                 f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)
             ]
-        print("Base directory: {}".format(base_directory))
-        print("{} Descriptors found to validate".format(len(descriptors_paths)))
+        self._logger.info("Base directory: {}".format(base_directory))
+        self._logger.info(
+            "{} Descriptors found to validate".format(len(descriptors_paths))
+        )
         for desc_path in descriptors_paths:
             with open(desc_path) as descriptor_file:
                 descriptor_data = descriptor_file.read()
             desc_type = "-"
             try:
+                # TODO: refactor validation_im.yaml_validation to @staticmethod
                 desc_type, descriptor_data = validation_im.yaml_validation(
                     self, descriptor_data
                 )
+                self._logger.debug(f"Validate {desc_type} {descriptor_data}")
                 if not old_format:
                     if desc_type == "vnfd" or desc_type == "nsd":
-                        print(
+                        self._logger.error(
                             "OSM descriptor '{}' written in an unsupported format. Please update to ETSI SOL006 format".format(
                                 desc_path
                             )
                         )
-                        print(
+                        self._logger.warning(
                             "Package validation skipped. It can still be done with 'osm package-validate --old'"
                         )
-                        print(
+                        self._logger.warning(
                             "Package build can still be done with 'osm package-build --skip-validation'"
                         )
                         raise Exception("Not SOL006 format")
@@ -178,6 +187,7 @@ class PackageTool(object):
                     {"type": desc_type, "path": desc_path, "valid": "OK", "error": "-"}
                 )
             except Exception as e:
+                self._logger.error(f"Validation error: {e}", exc_info=True)
                 table.append(
                     {
                         "type": desc_type,
@@ -186,6 +196,7 @@ class PackageTool(object):
                         "error": str(e),
                     }
                 )
+            self._logger.debug(table[-1])
         return table
 
     def translate(self, base_directory, recursive=True, dryrun=False):
@@ -495,25 +506,25 @@ class PackageTool(object):
         :return: Files and Folders not found. In case of override, it will return all file list
         """
         self._logger.debug("")
-        listCharms = []
+        charms_set = set()
         descriptor_file = False
         descriptors_paths = [f for f in glob.glob(package_folder + "/*.yaml")]
         for file in descriptors_paths:
             if file.endswith("nfd.yaml"):
                 descriptor_file = True
-                listCharms = self.charms_search(file, "vnf")
+                charms_set = self.charms_search(file, "vnf")
             if file.endswith("nsd.yaml"):
                 descriptor_file = True
-                listCharms = self.charms_search(file, "ns")
-        print("List of charms in the descriptor: {}".format(listCharms))
+                charms_set = self.charms_search(file, "ns")
+        print("List of charms in the descriptor: {}".format(charms_set))
         if not descriptor_file:
             raise ClientException(
                 'Descriptor filename is not correct in: {}. It should end with "nfd.yaml" or "nsd.yaml"'.format(
                     package_folder
                 )
             )
-        if listCharms and not skip_charm_build:
-            for charmName in listCharms:
+        if charms_set and not skip_charm_build:
+            for charmName in charms_set:
                 if os.path.isdir(
                     "{}/charms/layers/{}".format(package_folder, charmName)
                 ):
@@ -531,6 +542,8 @@ class PackageTool(object):
                 else:
                     if not os.path.isdir(
                         "{}/charms/{}".format(package_folder, charmName)
+                    ) and not os.path.isfile(
+                        "{}/charms/{}".format(package_folder, charmName)
                     ):
                         raise ClientException(
                             "The charm: {} referenced in the descriptor file "
@@ -538,8 +551,8 @@ class PackageTool(object):
                                 charmName, package_folder, package_folder
                             )
                         )
-        self._logger.debug("Return list of charms: {}".format(listCharms))
-        return listCharms
+        self._logger.debug("Return list of charms: {}".format(charms_set))
+        return charms_set
 
     def discover_folder_structure(self, base_directory, name, override):
         """
@@ -712,9 +725,12 @@ class PackageTool(object):
                             self._logger.debug(
                                 "Copying tree: {} -> {}".format(s_charm, d_temp)
                             )
-                            shutil.copytree(
-                                s_charm, d_temp, symlinks=True, ignore=ignore
-                            )
+                            if os.path.isdir(s_charm):
+                                shutil.copytree(
+                                    s_charm, d_temp, symlinks=True, ignore=ignore
+                                )
+                            else:
+                                shutil.copy2(s_charm, d_temp)
                             self._logger.debug("DONE")
                     else:
                         self._logger.debug("Copying tree: {} -> {}".format(s, d))
@@ -732,6 +748,7 @@ class PackageTool(object):
         self._logger.debug(
             "descriptor_file: {}, desc_type: {}".format(descriptor_file, desc_type)
         )
+        charms_set = set()
         with open("{}".format(descriptor_file)) as yaml_desc:
             descriptor_dict = yaml.safe_load(yaml_desc)
             # self._logger.debug("\n"+yaml.safe_dump(descriptor_dict, indent=4, default_flow_style=False))
@@ -749,7 +766,7 @@ class PackageTool(object):
                     or "nsd-catalog" in descriptor_dict
                 )
             ):
-                charms_list = self._charms_search_on_osm_im_dict(
+                charms_set = self._charms_search_on_osm_im_dict(
                     descriptor_dict, desc_type
                 )
             else:
@@ -759,12 +776,12 @@ class PackageTool(object):
                     get_charm_list = self._charms_search_on_vnfd_sol006_dict
                 else:
                     raise Exception("Bad descriptor type")
-                charms_list = get_charm_list(descriptor_dict)
-        return charms_list
+                charms_set = get_charm_list(descriptor_dict)
+        return charms_set
 
     def _charms_search_on_osm_im_dict(self, osm_im_dict, desc_type):
         self._logger.debug("")
-        charms_list = []
+        charms_set = []
         for k1, v1 in osm_im_dict.items():
             for k2, v2 in v1.items():
                 for entry in v2:
@@ -772,19 +789,19 @@ class PackageTool(object):
                         vnf_config = entry["{}-configuration".format(desc_type)]
                         for k3, v3 in vnf_config.items():
                             if "charm" in v3:
-                                charms_list.append((v3["charm"]))
+                                charms_set.add((v3["charm"]))
                     if "vdu" in entry:
                         vdus = entry["vdu"]
                         for vdu in vdus:
                             if "vdu-configuration" in vdu:
                                 for k4, v4 in vdu["vdu-configuration"].items():
                                     if "charm" in v4:
-                                        charms_list.append((v4["charm"]))
-        return charms_list
+                                        charms_set.add((v4["charm"]))
+        return charms_set
 
     def _charms_search_on_vnfd_sol006_dict(self, sol006_dict):
         self._logger.debug("")
-        charms_list = []
+        charms_set = set()
         dfs = sol006_dict.get("vnfd", {}).get("df", [])
         for df in dfs:
             day_1_2s = (
@@ -792,19 +809,20 @@ class PackageTool(object):
                 .get("operate-vnf-op-config", {})
                 .get("day1-2")
             )
-            for day_1_2 in day_1_2s:
-                exec_env_list = day_1_2.get("execution-environment-list", [])
-                for exec_env in exec_env_list:
-                    if "juju" in exec_env and "charm" in exec_env["juju"]:
-                        charms_list.append(exec_env["juju"]["charm"])
-        return charms_list
+            if day_1_2s is not None:
+                for day_1_2 in day_1_2s:
+                    exec_env_list = day_1_2.get("execution-environment-list", [])
+                    for exec_env in exec_env_list:
+                        if "juju" in exec_env and "charm" in exec_env["juju"]:
+                            charms_set.add(exec_env["juju"]["charm"])
+        return charms_set
 
     def _charms_search_on_nsd_sol006_dict(self, sol006_dict):
         self._logger.debug("")
-        charms_list = []
+        charms_set = set()
         nsd_list = sol006_dict.get("nsd", {}).get("nsd", [])
         for nsd in nsd_list:
             charm = nsd.get("ns-configuration", {}).get("juju", {}).get("charm")
             if charm:
-                charms_list.append(charm)
-        return charms_list
+                charms_set.add(charm)
+        return charms_set