Bug 1862: Fixed required field in SOL006 IM to generate OSM repo index

[osm/osmclient.git] / osmclient / sol005 / osmrepo.py

diff --git a/osmclient/sol005/osmrepo.py b/osmclient/sol005/osmrepo.py
index 424865c..85529de 100644 (file)
--- a/osmclient/sol005/osmrepo.py
+++ b/osmclient/sol005/osmrepo.py
@@ -16,7 +16,6 @@
 OSM Repo API handling
 """
 import glob
 OSM Repo API handling
 """
 import glob

-import hashlib

 import logging
 from os import listdir, mkdir, getcwd, remove
 from os.path import isfile, isdir, join, abspath
 import logging
 from os import listdir, mkdir, getcwd, remove
 from os.path import isfile, isdir, join, abspath


@@ -29,6 +28,7 @@ from osm_im.validation import Validation as validation_im
 from osmclient.common.exceptions import ClientException
 from osmclient.common.package_tool import PackageTool
 from osmclient.sol005.repo import Repo
 from osmclient.common.exceptions import ClientException
 from osmclient.common.package_tool import PackageTool
 from osmclient.sol005.repo import Repo

+from osmclient.common import utils

 from packaging import version as versioning
 import requests
 import yaml
 from packaging import version as versioning
 import requests
 import yaml


@@ -92,10 +92,11 @@ class OSMRepo(Repo):
                         "repository in url {} unreachable".format(repository.get("url"))
                     )
             except Exception as e:
                         "repository in url {} unreachable".format(repository.get("url"))
                     )
             except Exception as e:

-                logging.error(
+                self._logger.error(

                     "Error cannot read from repository {} '{}': {}".format(
                         repository["name"], repository["url"], e
                     "Error cannot read from repository {} '{}': {}".format(
                         repository["name"], repository["url"], e

-                    )
+                    ),
+                    exc_info=True,

                 )
                 continue
 
                 )
                 continue
 


@@ -170,97 +171,149 @@ class OSMRepo(Repo):
         :param origin: origin directory for getting all the artifacts
         :param destination: destination folder for create and index the valid artifacts
         """
         :param origin: origin directory for getting all the artifacts
         :param destination: destination folder for create and index the valid artifacts
         """

-        self._logger.debug("")
+        self._logger.debug("Starting index composition")

         if destination == ".":
             if origin == destination:
                 destination = "repository"
 
         destination = abspath(destination)
         origin = abspath(origin)
         if destination == ".":
             if origin == destination:
                 destination = "repository"
 
         destination = abspath(destination)
         origin = abspath(origin)

-
+        self._logger.debug(f"Paths {destination}, {origin}")

         if origin[0] != "/":
             origin = join(getcwd(), origin)
         if destination[0] != "/":
             destination = join(getcwd(), destination)
 
         self.init_directory(destination)
         if origin[0] != "/":
             origin = join(getcwd(), origin)
         if destination[0] != "/":
             destination = join(getcwd(), destination)
 
         self.init_directory(destination)

-        artifacts = [f for f in listdir(origin) if isfile(join(origin, f))]
-        directories = [f for f in listdir(origin) if isdir(join(origin, f))]
-        for artifact in artifacts:
-            self.register_artifact_in_repository(
-                join(origin, artifact), destination, source="file"
+        artifacts = []
+        directories = []
+        for f in listdir(origin):
+            self._logger.debug(f"Element: {join(origin,f)}")
+            if isfile(join(origin, f)) and f.endswith(".tar.gz"):
+                artifacts.append(f)
+            elif (
+                isdir(join(origin, f))
+                and f != destination.split("/")[-1]
+                and not f.startswith(".")
+            ):
+                directories.append(
+                    f
+                )  # TODO: Document that nested directories are not supported
+            else:
+                self._logger.debug(f"Ignoring {f}")
+        self._logger.debug(f"Artifacts: {artifacts}")
+        for package in artifacts:
+            self.register_package_in_repository(
+                join(origin, package), origin, destination, kind="artifact"

             )
             )

-        for artifact in directories:
-            self.register_artifact_in_repository(
-                join(origin, artifact), destination, source="directory"
+        self._logger.debug(f"Directories: {directories}")
+        for package in directories:
+            self.register_package_in_repository(
+                join(origin, package), origin, destination, kind="directory"

             )
             )

-        print("\nFinal Results: ")
-        print(
+        self._logger.info("\nFinal Results: ")
+        self._logger.info(

             "VNF Packages Indexed: "
             + str(len(glob.glob(destination + "/vnf/*/*/metadata.yaml")))
         )
             "VNF Packages Indexed: "
             + str(len(glob.glob(destination + "/vnf/*/*/metadata.yaml")))
         )

-        print(
+        self._logger.info(

             "NS Packages Indexed: "
             + str(len(glob.glob(destination + "/ns/*/*/metadata.yaml")))
         )
 
             "NS Packages Indexed: "
             + str(len(glob.glob(destination + "/ns/*/*/metadata.yaml")))
         )
 

-    def md5(self, fname):
-        """
-        Checksum generator
-        :param fname: file path
-        :return: checksum string
-        """
-        self._logger.debug("")
-        hash_md5 = hashlib.md5()
-        with open(fname, "rb") as f:
-            for chunk in iter(lambda: f.read(4096), b""):
-                hash_md5.update(chunk)
-        return hash_md5.hexdigest()
+        self._logger.info(
+            "NST Packages Indexed: "
+            + str(len(glob.glob(destination + "/nst/*/*/metadata.yaml")))
+        )

 
     def fields_building(self, descriptor_dict, file, package_type):
         """
         From an artifact descriptor, obtain the fields required for indexing
         :param descriptor_dict: artifact description
         :param file: artifact package
 
     def fields_building(self, descriptor_dict, file, package_type):
         """
         From an artifact descriptor, obtain the fields required for indexing
         :param descriptor_dict: artifact description
         :param file: artifact package

-        :param package_type: type of artifact (vnf or ns)
+        :param package_type: type of artifact (vnf, ns, nst)

         :return: fields
         """
         self._logger.debug("")
         :return: fields
         """
         self._logger.debug("")

+

         fields = {}
         base_path = "/{}/".format(package_type)
         aux_dict = {}
         if package_type == "vnf":
             if descriptor_dict.get("vnfd-catalog", False):
                 aux_dict = descriptor_dict.get("vnfd-catalog", {}).get("vnfd", [{}])[0]
         fields = {}
         base_path = "/{}/".format(package_type)
         aux_dict = {}
         if package_type == "vnf":
             if descriptor_dict.get("vnfd-catalog", False):
                 aux_dict = descriptor_dict.get("vnfd-catalog", {}).get("vnfd", [{}])[0]

-            else:
+            elif descriptor_dict.get("vnfd:vnfd-catalog"):

                 aux_dict = descriptor_dict.get("vnfd:vnfd-catalog", {}).get(
                     "vnfd", [{}]
                 )[0]
                 aux_dict = descriptor_dict.get("vnfd:vnfd-catalog", {}).get(
                     "vnfd", [{}]
                 )[0]

-
+            elif descriptor_dict.get("vnfd"):
+                aux_dict = descriptor_dict["vnfd"]
+                if aux_dict.get("vnfd"):
+                    aux_dict = aux_dict["vnfd"][0]
+            else:
+                msg = f"Unexpected descriptor format {descriptor_dict}"
+                self._logger.error(msg)
+                raise ValueError(msg)
+            self._logger.debug(
+                f"Extracted descriptor info for {package_type}: {aux_dict}"
+            )

             images = []
             images = []

-            for vdu in aux_dict.get("vdu", ()):
-                images.append(vdu.get("image"))
+            for vdu in aux_dict.get("vdu", aux_dict.get("kdu", ())):
+                images.append(vdu.get("image", vdu.get("name")))

             fields["images"] = images
             fields["images"] = images

-        if package_type == "ns":
+        elif package_type == "ns":

             if descriptor_dict.get("nsd-catalog", False):
                 aux_dict = descriptor_dict.get("nsd-catalog", {}).get("nsd", [{}])[0]
             if descriptor_dict.get("nsd-catalog", False):
                 aux_dict = descriptor_dict.get("nsd-catalog", {}).get("nsd", [{}])[0]

-            else:
+            elif descriptor_dict.get("nsd:nsd-catalog"):

                 aux_dict = descriptor_dict.get("nsd:nsd-catalog", {}).get("nsd", [{}])[
                     0
                 ]
                 aux_dict = descriptor_dict.get("nsd:nsd-catalog", {}).get("nsd", [{}])[
                     0
                 ]

-
+            elif descriptor_dict.get("nsd"):
+                aux_dict = descriptor_dict["nsd"]
+                if aux_dict.get("nsd"):
+                    aux_dict = descriptor_dict["nsd"]["nsd"][0]
+            else:
+                msg = f"Unexpected descriptor format {descriptor_dict}"
+                self._logger.error(msg)
+                raise ValueError(msg)

             vnfs = []
             vnfs = []

-
-            for vnf in aux_dict.get("constituent-vnfd", ()):
-                vnfs.append(vnf.get("vnfd-id-ref"))
+            if aux_dict.get("constituent-vnfd"):
+                for vnf in aux_dict.get("constituent-vnfd", ()):
+                    vnfs.append(vnf.get("vnfd-id-ref"))
+            else:
+                vnfs = aux_dict.get("vnfd-id")

             self._logger.debug("Used VNFS in the NSD: " + str(vnfs))
             fields["vnfd-id-ref"] = vnfs
             self._logger.debug("Used VNFS in the NSD: " + str(vnfs))
             fields["vnfd-id-ref"] = vnfs

-
-        fields["name"] = aux_dict.get("name")
+        elif package_type == "nst":
+            if descriptor_dict.get("nst-catalog", False):
+                aux_dict = descriptor_dict.get("nst-catalog", {}).get("nst", [{}])[0]
+            elif descriptor_dict.get("nst:nst-catalog"):
+                aux_dict = descriptor_dict.get("nst:nst-catalog", {}).get("nst", [{}])[
+                    0
+                ]
+            elif descriptor_dict.get("nst"):
+                aux_dict = descriptor_dict["nst"]
+                if aux_dict.get("nst"):
+                    aux_dict = descriptor_dict["nst"]["nst"][0]
+            nsds = []
+            for nsd in aux_dict.get("netslice-subnet", ()):
+                nsds.append(nsd.get("nsd-ref"))
+            self._logger.debug("Used NSDs in the NST: " + str(nsds))
+            if not nsds:
+                msg = f"Unexpected descriptor format {descriptor_dict}"
+                self._logger.error(msg)
+                raise ValueError(msg)
+            fields["nsd-id-ref"] = nsds
+        else:
+            msg = f"Unexpected descriptor format {descriptor_dict}"
+            self._logger.error(msg)
+            raise ValueError(msg)
+        # Repo search is based on 'name' entry in index.yaml. It is mandatory then
+        fields["name"] = aux_dict.get("name", aux_dict["product-name"])

         fields["id"] = aux_dict.get("id")
         fields["description"] = aux_dict.get("description")
         fields["vendor"] = aux_dict.get("vendor")
         fields["id"] = aux_dict.get("id")
         fields["description"] = aux_dict.get("description")
         fields["vendor"] = aux_dict.get("vendor")

-        fields["version"] = aux_dict.get("version", "1.0")
+        fields["version"] = str(aux_dict.get("version", "1.0"))

         fields["path"] = "{}{}/{}/{}-{}.tar.gz".format(
             base_path,
             fields["id"],
         fields["path"] = "{}{}/{}/{}-{}.tar.gz".format(
             base_path,
             fields["id"],


@@ -288,37 +341,51 @@ class OSMRepo(Repo):
         descriptor_file = glob.glob("{}/*.y*ml".format(folder))[0]
         return folder, descriptor_file
 
         descriptor_file = glob.glob("{}/*.y*ml".format(folder))[0]
         return folder, descriptor_file
 

-    def validate_artifact(self, path, source):
+    def validate_artifact(self, path, origin, kind):

         """
         Validation of artifact.
         :param path: file path
         """
         Validation of artifact.
         :param path: file path

+        :param origin: folder where the package is located
+        :param kind: flag to select the correct file type (directory or artifact)

         :return: status details, status, fields, package_type
         """
         :return: status details, status, fields, package_type
         """

-        self._logger.debug("")
+        self._logger.debug(f"Validating {path} {kind}")

         package_type = ""
         folder = ""
         try:
         package_type = ""
         folder = ""
         try:

-            if source == "directory":
+            if kind == "directory":

                 descriptor_file = glob.glob("{}/*.y*ml".format(path))[0]
             else:
                 folder, descriptor_file = self.zip_extraction(path)
                 descriptor_file = glob.glob("{}/*.y*ml".format(path))[0]
             else:
                 folder, descriptor_file = self.zip_extraction(path)

+                folder = join(origin, folder)
+                self._logger.debug(f"Kind is an artifact (tar.gz). Folder: {folder}. Descriptor_file: {descriptor_file}")

 
             self._logger.debug("Opening descriptor file: {}".format(descriptor_file))
 
             with open(descriptor_file, "r") as f:
                 descriptor_data = f.read()
 
             self._logger.debug("Opening descriptor file: {}".format(descriptor_file))
 
             with open(descriptor_file, "r") as f:
                 descriptor_data = f.read()

+            self._logger.debug(f"Descriptor data: {descriptor_data}")

             validation = validation_im()
             desc_type, descriptor_dict = validation.yaml_validation(descriptor_data)
             validation = validation_im()
             desc_type, descriptor_dict = validation.yaml_validation(descriptor_data)

-            validation_im.pyangbind_validation(self, desc_type, descriptor_dict)
-            if "vnf" in list(descriptor_dict.keys())[0]:
+            try:
+                validation_im.pyangbind_validation(self, desc_type, descriptor_dict)
+            except Exception as e:
+                self._logger.error(e, exc_info=True)
+                raise e
+            descriptor_type_ref = list(descriptor_dict.keys())[0].lower()
+            if "vnf" in descriptor_type_ref:

                 package_type = "vnf"
                 package_type = "vnf"

-            else:
-                # raise ClientException("Not VNF package")
+            elif "nst" in descriptor_type_ref:
+                package_type = "nst"
+            elif "ns" in descriptor_type_ref:

                 package_type = "ns"
                 package_type = "ns"

-
+            else:
+                msg = f"Unknown package type {descriptor_type_ref}"
+                self._logger.error(msg)
+                raise ValueError(msg)

             self._logger.debug("Descriptor: {}".format(descriptor_dict))
             fields = self.fields_building(descriptor_dict, path, package_type)
             self._logger.debug("Descriptor: {}".format(descriptor_dict))
             fields = self.fields_building(descriptor_dict, path, package_type)

-            self._logger.debug("Descriptor sucessfully validated")
+            self._logger.debug(f"Descriptor successfully validated {fields}")

             return (
                 {
                     "detail": "{}D successfully validated".format(package_type.upper()),
             return (
                 {
                     "detail": "{}D successfully validated".format(package_type.upper()),


@@ -335,37 +402,40 @@ class OSMRepo(Repo):
             if folder:
                 rmtree(folder, ignore_errors=True)
 
             if folder:
                 rmtree(folder, ignore_errors=True)
 

-    def register_artifact_in_repository(self, path, destination, source):
+    def register_package_in_repository(self, path, origin, destination, kind):

         """
         Registration of one artifact in a repository
         """
         Registration of one artifact in a repository

-        file: VNF or NS
-        destination: path for index creation
+        :param path: absolute path of the VNF/NS package
+        :param origin: folder where the package is located
+        :param destination: path for index creation
+        :param kind: artifact (tar.gz) or directory

         """
         self._logger.debug("")
         pt = PackageTool()
         """
         self._logger.debug("")
         pt = PackageTool()

-        compresed = False
+        compressed = False

         try:
             fields = {}
         try:
             fields = {}

-            _, valid, fields, package_type = self.validate_artifact(path, source)
+            _, valid, fields, package_type = self.validate_artifact(path, origin, kind)

             if not valid:
                 raise Exception(
                     "{} {} Not well configured.".format(package_type.upper(), str(path))
                 )
             else:
             if not valid:
                 raise Exception(
                     "{} {} Not well configured.".format(package_type.upper(), str(path))
                 )
             else:

-                if source == "directory":
+                if kind == "directory":

                     path = pt.build(path)
                     path = pt.build(path)

-                    compresed = True
-                fields["checksum"] = self.md5(path)
+                    self._logger.debug(f"Directory path {path}")
+                    compressed = True
+                fields["checksum"] = utils.md5(path)

                 self.indexation(destination, path, package_type, fields)
 
         except Exception as e:
             self._logger.exception(
                 self.indexation(destination, path, package_type, fields)
 
         except Exception as e:
             self._logger.exception(

-                "Error registering artifact in Repository: {}".format(e)
+                "Error registering package in Repository: {}".format(e)

             )
             raise ClientException(e)
 
         finally:
             )
             raise ClientException(e)
 
         finally:

-            if source == "directory" and compresed:
+            if kind == "directory" and compressed:

                 remove(path)
 
     def indexation(self, destination, path, package_type, fields):
                 remove(path)
 
     def indexation(self, destination, path, package_type, fields):


@@ -373,17 +443,18 @@ class OSMRepo(Repo):
         Process for index packages
         :param destination: index repository path
         :param path: path of the package
         Process for index packages
         :param destination: index repository path
         :param path: path of the package

-        :param package_type: package type (vnf, ns)
+        :param package_type: package type (vnf, ns, nst)

         :param fields: dict with the required values
         """
         :param fields: dict with the required values
         """

-        self._logger.debug("")
+        self._logger.debug(f"Processing {destination} {path} {package_type} {fields}")
+

         data_ind = {
             "name": fields.get("name"),
             "description": fields.get("description"),
             "vendor": fields.get("vendor"),
             "path": fields.get("path"),
         }
         data_ind = {
             "name": fields.get("name"),
             "description": fields.get("description"),
             "vendor": fields.get("vendor"),
             "path": fields.get("path"),
         }

-
+        self._logger.debug(data_ind)

         final_path = join(
             destination, package_type, fields.get("id"), fields.get("version")
         )
         final_path = join(
             destination, package_type, fields.get("id"), fields.get("version")
         )


@@ -473,7 +544,7 @@ class OSMRepo(Repo):
                 "{} {} added in the repository".format(package_type.upper(), str(path))
             )
 
                 "{} {} added in the repository".format(package_type.upper(), str(path))
             )
 

-    def current_datatime(self):
+    def current_datetime(self):

         """
         Datetime Generator
         :return: Datetime as string with the following structure "2020-04-29T08:41:07.681653Z"
         """
         Datetime Generator
         :return: Datetime as string with the following structure "2020-04-29T08:41:07.681653Z"


@@ -493,11 +564,13 @@ class OSMRepo(Repo):
         if not isfile(join(destination, "index.yaml")):
             mkdir(join(destination, "vnf"))
             mkdir(join(destination, "ns"))
         if not isfile(join(destination, "index.yaml")):
             mkdir(join(destination, "vnf"))
             mkdir(join(destination, "ns"))

+            mkdir(join(destination, "nst"))

             index_data = {
                 "apiVersion": "v1",
             index_data = {
                 "apiVersion": "v1",

-                "generated": self.current_datatime(),
+                "generated": self.current_datetime(),

                 "vnf_packages": {},
                 "ns_packages": {},
                 "vnf_packages": {},
                 "ns_packages": {},

+                "nst_packages": {},

             }
             with open(join(destination, "index.yaml"), "w") as outfile:
                 yaml.safe_dump(
             }
             with open(join(destination, "index.yaml"), "w") as outfile:
                 yaml.safe_dump(