osmclient/common/package_tool.py

   1 # /bin/env python3
   2 # Copyright 2019 ATOS
   3 #
   4 # All Rights Reserved.
   5 #
   6 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 #    not use this file except in compliance with the License. You may obtain
   8 #    a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #    Unless required by applicable law or agreed to in writing, software
  13 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 #    License for the specific language governing permissions and limitations
  16 #    under the License.
  17
  18 import glob
  19 import logging
  20 import os
  21 import shutil
  22 import subprocess
  23 import tarfile
  24 import time
  25 from jinja2 import Environment, PackageLoader
  26 from osm_im.validation import Validation as validation_im
  27 from osm_im.validation import ValidationException
  28 from osm_im import im_translation
  29 from osmclient.common import package_handling as package_handling
  30 from osmclient.common.exceptions import ClientException
  31 from osmclient.common import utils
  32 from .sol004_package import SOL004Package
  33 from .sol007_package import SOL007Package
  34 import yaml
  35
  36
  37 class PackageTool(object):
  38     def __init__(self, client=None):
  39         self._client = client
  40         self._logger = logging.getLogger("osmclient")
  41         self._validator = validation_im()
  42
  43     def create(
  44         self,
  45         package_type,
  46         base_directory,
  47         package_name,
  48         override,
  49         image,
  50         vdus,
  51         vcpu,
  52         memory,
  53         storage,
  54         interfaces,
  55         vendor,
  56         detailed,
  57         netslice_subnets,
  58         netslice_vlds,
  59         old,
  60     ):
  61         """
  62         **Create a package descriptor**
  63
  64         :params:
  65             - package_type: [vnf, ns, nst]
  66             - base directory: path of destination folder
  67             - package_name: is the name of the package to be created
  68             - image: specify the image of the vdu
  69             - vcpu: number of virtual cpus of the vdu
  70             - memory: amount of memory in MB pf the vdu
  71             - storage: amount of storage in GB of the vdu
  72             - interfaces: number of interfaces besides management interface
  73             - vendor: vendor name of the vnf/ns
  74             - detailed: include all possible values for NSD, VNFD, NST
  75             - netslice_subnets: number of netslice_subnets for the NST
  76             - netslice_vlds: number of virtual link descriptors for the NST
  77             - old: flag to create a descriptor using the previous OSM format (pre SOL006, OSM<9)
  78
  79         :return: status
  80         """
  81         self._logger.debug("")
  82         # print("location: {}".format(osmclient.__path__))
  83         file_loader = PackageLoader("osmclient")
  84         env = Environment(loader=file_loader, autoescape=True)
  85         if package_type == "ns":
  86             template = env.get_template("nsd.yaml.j2" if not old else "nsd_old.yaml.j2")
  87             content = {
  88                 "name": package_name,
  89                 "vendor": vendor,
  90                 "vdus": vdus,
  91                 "clean": False,
  92                 "interfaces": interfaces,
  93                 "detailed": detailed,
  94             }
  95         elif package_type == "vnf":
  96             template = env.get_template(
  97                 "vnfd.yaml.j2" if not old else "vnfd_old.yaml.j2"
  98             )
  99             content = {
 100                 "name": package_name,
 101                 "vendor": vendor,
 102                 "vdus": vdus,
 103                 "clean": False,
 104                 "interfaces": interfaces,
 105                 "image": image,
 106                 "vcpu": vcpu,
 107                 "memory": memory,
 108                 "storage": storage,
 109                 "detailed": detailed,
 110             }
 111         elif package_type == "nst":
 112             # TODO: repo-index did not support nst in OSM<9, no changes in template
 113             template = env.get_template("nst.yaml.j2")
 114             content = {
 115                 "name": package_name,
 116                 "vendor": vendor,
 117                 "interfaces": interfaces,
 118                 "netslice_subnets": netslice_subnets,
 119                 "netslice_vlds": netslice_vlds,
 120                 "detailed": detailed,
 121             }
 122         else:
 123             raise ClientException(
 124                 "Wrong descriptor type {}. Options: ns, vnf, nst".format(package_type)
 125             )
 126
 127         self._logger.debug("To be rendered: {}".format(content))
 128         output = template.render(content)
 129         self._logger.debug(output)
 130
 131         structure = self.discover_folder_structure(
 132             base_directory, package_name, override
 133         )
 134         if structure.get("folders"):
 135             self.create_folders(structure["folders"], package_type)
 136         if structure.get("files"):
 137             self.create_files(structure["files"], output, package_type)
 138         return "Created"
 139
 140     def validate(self, base_directory, recursive=True, old_format=False):
 141         """
 142         **Validate OSM Descriptors given a path**
 143
 144         :params:
 145             - base_directory is the root path for all descriptors
 146
 147         :return: List of dict of validated descriptors. keys: type, path, valid, error
 148         """
 149         self._logger.debug("")
 150         table = []
 151         if recursive:
 152             descriptors_paths = [
 153                 f for f in glob.glob(base_directory + "/**/*.yaml", recursive=recursive)
 154             ]
 155         else:
 156             descriptors_paths = [
 157                 f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)
 158             ]
 159         self._logger.info("Base directory: {}".format(base_directory))
 160         self._logger.info(
 161             "{} Descriptors found to validate".format(len(descriptors_paths))
 162         )
 163         for desc_path in descriptors_paths:
 164             with open(desc_path) as descriptor_file:
 165                 descriptor_data = descriptor_file.read()
 166             desc_type = "-"
 167             try:
 168                 # TODO: refactor validation_im.yaml_validation to @staticmethod
 169                 desc_type, descriptor_data = validation_im.yaml_validation(
 170                     self, descriptor_data
 171                 )
 172                 self._logger.debug(f"Validate {desc_type} {descriptor_data}")
 173                 if not old_format:
 174                     if desc_type == "vnfd" or desc_type == "nsd":
 175                         self._logger.error(
 176                             "OSM descriptor '{}' written in an unsupported format. Please update to ETSI SOL006 format".format(
 177                                 desc_path
 178                             )
 179                         )
 180                         self._logger.warning(
 181                             "Package validation skipped. It can still be done with 'osm package-validate --old'"
 182                         )
 183                         self._logger.warning(
 184                             "Package build can still be done with 'osm package-build --skip-validation'"
 185                         )
 186                         raise Exception("Not SOL006 format")
 187                 validation_im.pyangbind_validation(self, desc_type, descriptor_data)
 188                 table.append(
 189                     {"type": desc_type, "path": desc_path, "valid": "OK", "error": "-"}
 190                 )
 191             except Exception as e:
 192                 self._logger.error(f"Validation error: {e}", exc_info=True)
 193                 table.append(
 194                     {
 195                         "type": desc_type,
 196                         "path": desc_path,
 197                         "valid": "ERROR",
 198                         "error": str(e),
 199                     }
 200                 )
 201             self._logger.debug(table[-1])
 202         return table
 203
 204     def translate(self, base_directory, recursive=True, dryrun=False):
 205         """
 206         **Translate OSM Packages given a path**
 207
 208         :params:
 209             - base_directory is the root path for all packages
 210
 211         :return: List of dict of translated packages. keys: current type, new type, path, valid, translated, error
 212         """
 213         self._logger.debug("")
 214         table = []
 215         if recursive:
 216             descriptors_paths = [
 217                 f for f in glob.glob(base_directory + "/**/*.yaml", recursive=recursive)
 218             ]
 219         else:
 220             descriptors_paths = [
 221                 f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)
 222             ]
 223         print("Base directory: {}".format(base_directory))
 224         print("{} Descriptors found to validate".format(len(descriptors_paths)))
 225         for desc_path in descriptors_paths:
 226             with open(desc_path) as descriptor_file:
 227                 descriptor_data = descriptor_file.read()
 228             desc_type = "-"
 229             try:
 230                 desc_type, descriptor_data = validation_im.yaml_validation(
 231                     self, descriptor_data
 232                 )
 233                 self._logger.debug("desc_type: {}".format(desc_type))
 234                 self._logger.debug("descriptor_data:\n{}".format(descriptor_data))
 235                 self._validator.pyangbind_validation(desc_type, descriptor_data)
 236                 if not (desc_type == "vnfd" or desc_type == "nsd"):
 237                     table.append(
 238                         {
 239                             "current type": desc_type,
 240                             "new type": desc_type,
 241                             "path": desc_path,
 242                             "valid": "OK",
 243                             "translated": "N/A",
 244                             "error": "-",
 245                         }
 246                     )
 247                 else:
 248                     new_desc_type = desc_type
 249                     try:
 250                         sol006_model = yaml.safe_dump(
 251                             im_translation.translate_im_model_to_sol006(
 252                                 descriptor_data
 253                             ),
 254                             indent=4,
 255                             default_flow_style=False,
 256                         )
 257                         (
 258                             new_desc_type,
 259                             new_descriptor_data,
 260                         ) = self._validator.yaml_validation(sol006_model)
 261                         self._validator.pyangbind_validation(
 262                             new_desc_type, new_descriptor_data
 263                         )
 264                         if not dryrun:
 265                             with open(desc_path, "w") as descriptor_file:
 266                                 descriptor_file.write(sol006_model)
 267                         table.append(
 268                             {
 269                                 "current type": desc_type,
 270                                 "new type": new_desc_type,
 271                                 "path": desc_path,
 272                                 "valid": "OK",
 273                                 "translated": "OK",
 274                                 "error": "-",
 275                             }
 276                         )
 277                     except ValidationException as ve2:
 278                         table.append(
 279                             {
 280                                 "current type": desc_type,
 281                                 "new type": new_desc_type,
 282                                 "path": desc_path,
 283                                 "valid": "OK",
 284                                 "translated": "ERROR",
 285                                 "error": "Error in the post-validation: {}".format(
 286                                     str(ve2)
 287                                 ),
 288                             }
 289                         )
 290                     except Exception as e2:
 291                         table.append(
 292                             {
 293                                 "current type": desc_type,
 294                                 "new type": new_desc_type,
 295                                 "path": desc_path,
 296                                 "valid": "OK",
 297                                 "translated": "ERROR",
 298                                 "error": "Error in the translation: {}".format(str(e2)),
 299                             }
 300                         )
 301             except ValidationException as ve:
 302                 table.append(
 303                     {
 304                         "current type": desc_type,
 305                         "new type": "N/A",
 306                         "path": desc_path,
 307                         "valid": "ERROR",
 308                         "translated": "N/A",
 309                         "error": "Error in the pre-validation: {}".format(str(ve)),
 310                     }
 311                 )
 312             except Exception as e:
 313                 table.append(
 314                     {
 315                         "current type": desc_type,
 316                         "new type": "N/A",
 317                         "path": desc_path,
 318                         "valid": "ERROR",
 319                         "translated": "N/A",
 320                         "error": str(e),
 321                     }
 322                 )
 323         return table
 324
 325     def descriptor_translate(self, descriptor_file):
 326         """
 327         **Translate input descriptor file from Rel EIGHT OSM to SOL006**
 328
 329         :params:
 330             - base_directory is the root path for all packages
 331
 332         :return: YAML descriptor in the new format
 333         """
 334         self._logger.debug("")
 335         with open(descriptor_file, "r") as df:
 336             im_model = yaml.safe_load(df.read())
 337         sol006_model = im_translation.translate_im_model_to_sol006(im_model)
 338         return yaml.safe_dump(sol006_model, indent=4, default_flow_style=False)
 339
 340     def build(self, package_folder, skip_validation=False, skip_charm_build=False):
 341         """
 342         **Creates a .tar.gz file given a package_folder**
 343
 344         :params:
 345             - package_folder: is the name of the folder to be packaged
 346             - skip_validation: is the flag to validate or not the descriptors on the folder before build
 347
 348         :returns: message result for the build process
 349         """
 350         self._logger.debug("")
 351         package_folder = package_folder.rstrip("/")
 352         if not os.path.exists("{}".format(package_folder)):
 353             return "Fail, package is not in the specified path"
 354         if not skip_validation:
 355             print("Validating package {}".format(package_folder))
 356             results = self.validate(package_folder, recursive=False)
 357             if results:
 358                 for result in results:
 359                     if result["valid"] != "OK":
 360                         raise ClientException(
 361                             "There was an error validating the file {} with error: {}".format(
 362                                 result["path"], result["error"]
 363                             )
 364                         )
 365                 print("Validation OK")
 366             else:
 367                 raise ClientException(
 368                     "No descriptor file found in: {}".format(package_folder)
 369                 )
 370
 371         is_sol004_007 = (
 372             package_handling.get_package_type(package_folder)
 373             != package_handling.OSM_OLD
 374         )
 375
 376         charm_list = self.build_all_charms(
 377             package_folder, skip_charm_build, is_sol004_007
 378         )
 379         return self.build_compressed_file(package_folder, charm_list, is_sol004_007)
 380
 381     def calculate_checksum(self, package_folder):
 382         """
 383         **Function to calculate the checksum given a folder**
 384
 385         :params:
 386             - package_folder: is the folder where we have the files to calculate the checksum
 387         :returns: None
 388         """
 389         self._logger.debug("")
 390         files = [
 391             f
 392             for f in glob.glob(package_folder + "/**/*.*", recursive=True)
 393             if os.path.isfile(f)
 394         ]
 395         with open("{}/checksums.txt".format(package_folder), "w+") as checksum:
 396             for file_item in files:
 397                 if "checksums.txt" in file_item:
 398                     continue
 399                 checksum.write("{}\t{}\n".format(utils.md5(file_item), file_item))
 400
 401     def create_folders(self, folders, package_type):
 402         """
 403         **Create folder given a list of folders**
 404
 405         :params:
 406             - folders: [List] list of folders paths to be created
 407             - package_type: is the type of package to be created
 408         :return: None
 409         """
 410         self._logger.debug("")
 411         for folder in folders:
 412             try:
 413                 # print("Folder {} == package_type {}".format(folder[1], package_type))
 414                 if folder[1] == package_type:
 415                     print("Creating folder:\t{}".format(folder[0]))
 416                     os.makedirs(folder[0])
 417             except FileExistsError:
 418                 pass
 419
 420     def save_file(self, file_name, file_body):
 421         """
 422         **Create a file given a name and the content**
 423
 424         :params:
 425             - file_name: is the name of the file with the relative route
 426             - file_body: is the content of the file
 427         :return: None
 428         """
 429         self._logger.debug("")
 430         print("Creating file:  \t{}".format(file_name))
 431         try:
 432             with open(file_name, "w+") as f:
 433                 f.write(file_body)
 434         except Exception as e:
 435             raise ClientException(e)
 436
 437     def generate_readme(self):
 438         """
 439         **Creates the README content**
 440
 441         :returns: readme content
 442         """
 443         self._logger.debug("")
 444         return """# Descriptor created by OSM descriptor package generated\n\n**Created on {} **""".format(
 445             time.strftime("%m/%d/%Y, %H:%M:%S", time.localtime())
 446         )
 447
 448     def generate_cloud_init(self):
 449         """
 450         **Creates the cloud-init content**
 451
 452         :returns: cloud-init content
 453         """
 454         self._logger.debug("")
 455         return "---\n#cloud-config"
 456
 457     def create_files(self, files, file_content, package_type):
 458         """
 459         **Creates the files given the file list and type**
 460
 461         :params:
 462             - files: is the list of files structure
 463             - file_content: is the content of the descriptor rendered by the template
 464             - package_type: is the type of package to filter the creation structure
 465
 466         :return: None
 467         """
 468         self._logger.debug("")
 469         for file_item, file_package, file_type in files:
 470             if package_type == file_package:
 471                 if file_type == "descriptor":
 472                     self.save_file(file_item, file_content)
 473                 elif file_type == "readme":
 474                     self.save_file(file_item, self.generate_readme())
 475                 elif file_type == "cloud_init":
 476                     self.save_file(file_item, self.generate_cloud_init())
 477
 478     def check_files_folders(self, path_list, override):
 479         """
 480         **Find files and folders missing given a directory structure {"folders": [], "files": []}**
 481
 482         :params:
 483             - path_list: is the list of files and folders to be created
 484             - override: is the flag used to indicate the creation of the list even if the file exist to override it
 485
 486         :return: Missing paths Dict
 487         """
 488         self._logger.debug("")
 489         missing_paths = {}
 490         folders = []
 491         files = []
 492         for folder in path_list.get("folders"):
 493             if not os.path.exists(folder[0]):
 494                 folders.append(folder)
 495         missing_paths["folders"] = folders
 496
 497         for file_item in path_list.get("files"):
 498             if not os.path.exists(file_item[0]) or override is True:
 499                 files.append(file_item)
 500         missing_paths["files"] = files
 501
 502         return missing_paths
 503
 504     def build_all_charms(self, package_folder, skip_charm_build, sol004_007=True):
 505         """
 506         **Read the descriptor file, check that the charms referenced are in the folder and compiles them**
 507
 508         :params:
 509             - packet_folder: is the location of the package
 510         :return: Files and Folders not found. In case of override, it will return all file list
 511         """
 512         self._logger.debug("")
 513         charms_set = set()
 514         descriptor_file = False
 515         package_type = package_handling.get_package_type(package_folder)
 516         if sol004_007 and package_type.find("TOSCA") >= 0:
 517             descriptors_paths = [
 518                 f for f in glob.glob(package_folder + "/Definitions/*.yaml")
 519             ]
 520         else:
 521             descriptors_paths = [f for f in glob.glob(package_folder + "/*.yaml")]
 522         for file in descriptors_paths:
 523             if file.endswith("nfd.yaml"):
 524                 descriptor_file = True
 525                 charms_set = self.charms_search(file, "vnf")
 526             if file.endswith("nsd.yaml"):
 527                 descriptor_file = True
 528                 charms_set = self.charms_search(file, "ns")
 529         print("List of charms in the descriptor: {}".format(charms_set))
 530         if not descriptor_file:
 531             raise ClientException(
 532                 'Descriptor filename is not correct in: {}. It should end with "nfd.yaml" or "nsd.yaml"'.format(
 533                     package_folder
 534                 )
 535             )
 536         if charms_set and not skip_charm_build:
 537             for charmName in charms_set:
 538                 if os.path.isdir(
 539                     "{}/{}charms/layers/{}".format(
 540                         package_folder, "Scripts/" if sol004_007 else "", charmName
 541                     )
 542                 ):
 543                     print(
 544                         "Building charm {}/{}charms/layers/{}".format(
 545                             package_folder, "Scripts/" if sol004_007 else "", charmName
 546                         )
 547                     )
 548                     self.charm_build(package_folder, charmName, sol004_007)
 549                     print("Charm built: {}".format(charmName))
 550                 elif os.path.isdir(
 551                     "{}/{}charms/ops/{}".format(
 552                         package_folder, "Scripts/" if sol004_007 else "", charmName
 553                     )
 554                 ):
 555                     self.charmcraft_build(package_folder, charmName)
 556                 else:
 557                     if not os.path.isdir(
 558                         "{}/{}charms/{}".format(
 559                             package_folder, "Scripts/" if sol004_007 else "", charmName
 560                         )
 561                     ) and not os.path.isfile(
 562                         "{}/{}charms/{}".format(
 563                             package_folder, "Scripts/" if sol004_007 else "", charmName
 564                         )
 565                     ):
 566                         raise ClientException(
 567                             "The charm: {} referenced in the descriptor file "
 568                             "is not present either in {}/charms or in {}/charms/layers".format(
 569                                 charmName, package_folder, package_folder
 570                             )
 571                         )
 572         self._logger.debug("Return list of charms: {}".format(charms_set))
 573         return charms_set
 574
 575     def discover_folder_structure(self, base_directory, name, override):
 576         """
 577         **Discover files and folders structure for SOL004/SOL007 descriptors given a base_directory and name**
 578
 579         :params:
 580             - base_directory: is the location of the package to be created
 581             - name: is the name of the package
 582             - override: is the flag used to indicate the creation of the list even if the file exist to override it
 583         :return: Files and Folders not found. In case of override, it will return all file list
 584         """
 585         self._logger.debug("")
 586         prefix = "{}/{}".format(base_directory, name)
 587         files_folders = {
 588             "folders": [
 589                 ("{}_ns".format(prefix), "ns"),
 590                 ("{}_ns/Licenses".format(prefix), "ns"),
 591                 ("{}_ns/Files/icons".format(prefix), "ns"),
 592                 ("{}_ns/Scripts/charms".format(prefix), "ns"),
 593                 ("{}_vnf".format(name), "vnf"),
 594                 ("{}_vnf/Licenses".format(prefix), "vnf"),
 595                 ("{}_vnf/Scripts/charms".format(prefix), "vnf"),
 596                 ("{}_vnf/Scripts/cloud_init".format(prefix), "vnf"),
 597                 ("{}_vnf/Files/images".format(prefix), "vnf"),
 598                 ("{}_vnf/Files/icons".format(prefix), "vnf"),
 599                 ("{}_vnf/Scripts/scripts".format(prefix), "vnf"),
 600                 ("{}_nst".format(prefix), "nst"),
 601                 ("{}_nst/icons".format(prefix), "nst"),
 602             ],
 603             "files": [
 604                 ("{}_ns/{}_nsd.yaml".format(prefix, name), "ns", "descriptor"),
 605                 ("{}_ns/README.md".format(prefix), "ns", "readme"),
 606                 ("{}_vnf/{}_vnfd.yaml".format(prefix, name), "vnf", "descriptor"),
 607                 (
 608                     "{}_vnf/Scripts/cloud_init/cloud-config.txt".format(prefix),
 609                     "vnf",
 610                     "cloud_init",
 611                 ),
 612                 ("{}_vnf/README.md".format(prefix), "vnf", "readme"),
 613                 ("{}_nst/{}_nst.yaml".format(prefix, name), "nst", "descriptor"),
 614                 ("{}_nst/README.md".format(prefix), "nst", "readme"),
 615             ],
 616         }
 617         missing_files_folders = self.check_files_folders(files_folders, override)
 618         # print("Missing files and folders: {}".format(missing_files_folders))
 619         return missing_files_folders
 620
 621     def charm_build(self, charms_folder, build_name, sol004_007=True):
 622         """
 623         Build the charms inside the package.
 624         params: package_folder is the name of the folder where is the charms to compile.
 625                 build_name is the name of the layer or interface
 626         """
 627         self._logger.debug("")
 628
 629         if sol004_007:
 630             os.environ["JUJU_REPOSITORY"] = "{}/Scripts/charms".format(charms_folder)
 631         else:
 632             os.environ["JUJU_REPOSITORY"] = "{}/charms".format(charms_folder)
 633
 634         os.environ["CHARM_LAYERS_DIR"] = "{}/layers".format(
 635             os.environ["JUJU_REPOSITORY"]
 636         )
 637         os.environ["CHARM_INTERFACES_DIR"] = "{}/interfaces".format(
 638             os.environ["JUJU_REPOSITORY"]
 639         )
 640
 641         if sol004_007:
 642             os.environ["CHARM_BUILD_DIR"] = "{}/Scripts/charms/builds".format(
 643                 charms_folder
 644             )
 645         else:
 646             os.environ["CHARM_BUILD_DIR"] = "{}/charms/builds".format(charms_folder)
 647
 648         if not os.path.exists(os.environ["CHARM_BUILD_DIR"]):
 649             os.makedirs(os.environ["CHARM_BUILD_DIR"])
 650         src_folder = "{}/{}".format(os.environ["CHARM_LAYERS_DIR"], build_name)
 651         result = subprocess.run(["charm", "build", "{}".format(src_folder)])
 652         if result.returncode == 1:
 653             raise ClientException("failed to build the charm: {}".format(src_folder))
 654         self._logger.verbose("charm {} built".format(src_folder))
 655
 656     def charmcraft_build(self, package_folder, charm_name):
 657         """
 658         Build the charms inside the package (new operator framework charms)
 659         params: package_folder is the name of the folder where is the charms to compile.
 660                 build_name is the name of the layer or interface
 661         """
 662         self._logger.debug("Building charm {}".format(charm_name))
 663         src_folder = f"{package_folder}/Scripts/charms/ops/{charm_name}"
 664         current_directory = os.getcwd()
 665         os.chdir(src_folder)
 666         try:
 667             result = subprocess.run(["charmcraft", "build"])
 668             if result.returncode == 1:
 669                 raise ClientException(
 670                     "failed to build the charm: {}".format(src_folder)
 671                 )
 672             subprocess.run(["rm", "-rf", f"../../{charm_name}"])
 673             subprocess.run(["mv", "build", f"../../{charm_name}"])
 674             self._logger.verbose("charm {} built".format(src_folder))
 675         finally:
 676             os.chdir(current_directory)
 677
 678     def build_compressed_file(self, package_folder, charm_list=None, sol004_007=True):
 679         if sol004_007:
 680             return self.build_zipfile(package_folder, charm_list)
 681         else:
 682             return self.build_tarfile(package_folder, charm_list)
 683
 684     def build_zipfile(self, package_folder, charm_list=None):
 685         """
 686         Creates a zip file given a package_folder
 687         params: package_folder is the name of the folder to be packaged
 688         returns: .zip name
 689         """
 690         self._logger.debug("")
 691         cwd = None
 692         try:
 693             directory_name, package_name = self.create_temp_dir_sol004_007(
 694                 package_folder, charm_list
 695             )
 696             cwd = os.getcwd()
 697             os.chdir(directory_name)
 698             package_type = package_handling.get_package_type(package_folder)
 699             print(package_type)
 700
 701             if (
 702                 package_handling.SOL007 == package_type
 703                 or package_handling.SOL007_TOSCA == package_type
 704             ):
 705                 the_package = SOL007Package(package_folder)
 706             elif (
 707                 package_handling.SOL004 == package_type
 708                 or package_handling.SOL004_TOSCA == package_type
 709             ):
 710                 the_package = SOL004Package(package_folder)
 711             else:
 712                 raise ClientException(f"Unknown package type: {package_type}")
 713
 714             the_package.create_or_update_metadata_file()
 715
 716             the_zip_package = shutil.make_archive(
 717                 os.path.join(cwd, package_name),
 718                 "zip",
 719                 os.path.join(directory_name, package_name),
 720             )
 721
 722             print("Package created: {}".format(the_zip_package))
 723
 724             return the_zip_package
 725
 726         except Exception as exc:
 727             raise ClientException(
 728                 "failure during build of zip file (create temp dir, calculate checksum, "
 729                 "zip file): {}".format(exc)
 730             )
 731         finally:
 732             if cwd:
 733                 os.chdir(cwd)
 734             shutil.rmtree(os.path.join(package_folder, "tmp"))
 735
 736     def build_tarfile(self, package_folder, charm_list=None):
 737         """
 738         Creates a .tar.gz file given a package_folder
 739         params: package_folder is the name of the folder to be packaged
 740         returns: .tar.gz name
 741         """
 742         self._logger.debug("")
 743         cwd = None
 744         try:
 745             directory_name, package_name = self.create_temp_dir(
 746                 package_folder, charm_list
 747             )
 748             cwd = os.getcwd()
 749             os.chdir(directory_name)
 750             self.calculate_checksum(package_name)
 751             with tarfile.open("{}.tar.gz".format(package_name), mode="w:gz") as archive:
 752                 print("Adding File: {}".format(package_name))
 753                 archive.add("{}".format(package_name), recursive=True)
 754             # return "Created {}.tar.gz".format(package_folder)
 755             # self.build("{}".format(os.path.basename(package_folder)))
 756             os.chdir(cwd)
 757             cwd = None
 758             created_package = "{}/{}.tar.gz".format(
 759                 os.path.dirname(package_folder) or ".", package_name
 760             )
 761             os.rename(
 762                 "{}/{}.tar.gz".format(directory_name, package_name), created_package
 763             )
 764             os.rename(
 765                 "{}/{}/checksums.txt".format(directory_name, package_name),
 766                 "{}/checksums.txt".format(package_folder),
 767             )
 768             print("Package created: {}".format(created_package))
 769             return created_package
 770         except Exception as exc:
 771             raise ClientException(
 772                 "failure during build of targz file (create temp dir, calculate checksum, "
 773                 "tar.gz file): {}".format(exc)
 774             )
 775         finally:
 776             if cwd:
 777                 os.chdir(cwd)
 778             shutil.rmtree(os.path.join(package_folder, "tmp"))
 779
 780     def create_temp_dir(self, package_folder, charm_list=None):
 781         """
 782         Method to create a temporary folder where we can move the files in package_folder
 783         """
 784         self._logger.debug("")
 785         ignore_patterns = ".gitignore"
 786         ignore = shutil.ignore_patterns(ignore_patterns)
 787         directory_name = os.path.abspath(package_folder)
 788         package_name = os.path.basename(directory_name)
 789         directory_name += "/tmp"
 790         os.makedirs("{}/{}".format(directory_name, package_name), exist_ok=True)
 791         self._logger.debug("Makedirs DONE: {}/{}".format(directory_name, package_name))
 792         for item in os.listdir(package_folder):
 793             self._logger.debug("Item: {}".format(item))
 794             if item != "tmp":
 795                 s = os.path.join(package_folder, item)
 796                 d = os.path.join(os.path.join(directory_name, package_name), item)
 797                 if os.path.isdir(s):
 798                     if item == "charms":
 799                         os.makedirs(d, exist_ok=True)
 800                         s_builds = os.path.join(s, "builds")
 801                         for charm in charm_list:
 802                             self._logger.debug("Copying charm {}".format(charm))
 803                             if charm in os.listdir(s):
 804                                 s_charm = os.path.join(s, charm)
 805                             elif charm in os.listdir(s_builds):
 806                                 s_charm = os.path.join(s_builds, charm)
 807                             else:
 808                                 raise ClientException(
 809                                     "The charm {} referenced in the descriptor file "
 810                                     "could not be found in {}/charms or in {}/charms/builds".format(
 811                                         charm, package_folder, package_folder
 812                                     )
 813                                 )
 814                             d_temp = os.path.join(d, charm)
 815                             self._logger.debug(
 816                                 "Copying tree: {} -> {}".format(s_charm, d_temp)
 817                             )
 818                             if os.path.isdir(s_charm):
 819                                 shutil.copytree(
 820                                     s_charm, d_temp, symlinks=True, ignore=ignore
 821                                 )
 822                             else:
 823                                 shutil.copy2(s_charm, d_temp)
 824                             self._logger.debug("DONE")
 825                     else:
 826                         self._logger.debug("Copying tree: {} -> {}".format(s, d))
 827                         shutil.copytree(s, d, symlinks=True, ignore=ignore)
 828                         self._logger.debug("DONE")
 829                 else:
 830                     if item in ignore_patterns:
 831                         continue
 832                     self._logger.debug("Copying file: {} -> {}".format(s, d))
 833                     shutil.copy2(s, d)
 834                     self._logger.debug("DONE")
 835         return directory_name, package_name
 836
 837     def copy_tree(self, s, d, ignore):
 838         self._logger.debug("Copying tree: {} -> {}".format(s, d))
 839         shutil.copytree(s, d, symlinks=True, ignore=ignore)
 840         self._logger.debug("DONE")
 841
 842     def create_temp_dir_sol004_007(self, package_folder, charm_list=None):
 843         """
 844         Method to create a temporary folder where we can move the files in package_folder
 845         """
 846         self._logger.debug("")
 847         ignore_patterns = ".gitignore"
 848         ignore = shutil.ignore_patterns(ignore_patterns)
 849         directory_name = os.path.abspath(package_folder)
 850         package_name = os.path.basename(directory_name)
 851         directory_name += "/tmp"
 852         os.makedirs("{}/{}".format(directory_name, package_name), exist_ok=True)
 853         self._logger.debug("Makedirs DONE: {}/{}".format(directory_name, package_name))
 854         for item in os.listdir(package_folder):
 855             self._logger.debug("Item: {}".format(item))
 856             if item != "tmp":
 857                 s = os.path.join(package_folder, item)
 858                 d = os.path.join(os.path.join(directory_name, package_name), item)
 859                 if os.path.isdir(s):
 860                     if item == "Scripts":
 861                         os.makedirs(d, exist_ok=True)
 862                         scripts_folder = s
 863                         for script_item in os.listdir(scripts_folder):
 864                             scripts_destination_folder = os.path.join(d, script_item)
 865                             if script_item == "charms":
 866                                 s_builds = os.path.join(
 867                                     scripts_folder, script_item, "builds"
 868                                 )
 869                                 for charm in charm_list:
 870                                     self._logger.debug("Copying charm {}".format(charm))
 871                                     if charm in os.listdir(
 872                                         os.path.join(scripts_folder, script_item)
 873                                     ):
 874                                         s_charm = os.path.join(
 875                                             scripts_folder, script_item, charm
 876                                         )
 877                                     elif charm in os.listdir(s_builds):
 878                                         s_charm = os.path.join(s_builds, charm)
 879                                     else:
 880                                         raise ClientException(
 881                                             "The charm {} referenced in the descriptor file "
 882                                             "could not be found in {}/charms or in {}/charms/builds".format(
 883                                                 charm, package_folder, package_folder
 884                                             )
 885                                         )
 886                                     d_temp = os.path.join(
 887                                         scripts_destination_folder, charm
 888                                     )
 889                                     self.copy_tree(s_charm, d_temp, ignore)
 890                             else:
 891                                 self.copy_tree(
 892                                     os.path.join(scripts_folder, script_item),
 893                                     scripts_destination_folder,
 894                                     ignore,
 895                                 )
 896                     else:
 897                         self.copy_tree(s, d, ignore)
 898                 else:
 899                     if item in ignore_patterns:
 900                         continue
 901                     self._logger.debug("Copying file: {} -> {}".format(s, d))
 902                     shutil.copy2(s, d)
 903                     self._logger.debug("DONE")
 904         return directory_name, package_name
 905
 906     def charms_search(self, descriptor_file, desc_type):
 907         self._logger.debug(
 908             "descriptor_file: {}, desc_type: {}".format(descriptor_file, desc_type)
 909         )
 910         charms_set = set()
 911         with open("{}".format(descriptor_file)) as yaml_desc:
 912             descriptor_dict = yaml.safe_load(yaml_desc)
 913             # self._logger.debug("\n"+yaml.safe_dump(descriptor_dict, indent=4, default_flow_style=False))
 914
 915             if (
 916                 desc_type == "vnf"
 917                 and (
 918                     "vnfd:vnfd-catalog" in descriptor_dict
 919                     or "vnfd-catalog" in descriptor_dict
 920                 )
 921             ) or (
 922                 desc_type == "ns"
 923                 and (
 924                     "nsd:nsd-catalog" in descriptor_dict
 925                     or "nsd-catalog" in descriptor_dict
 926                 )
 927             ):
 928                 charms_set = self._charms_search_on_osm_im_dict(
 929                     descriptor_dict, desc_type
 930                 )
 931             else:
 932                 if desc_type == "ns":
 933                     get_charm_list = self._charms_search_on_nsd_sol006_dict
 934                 elif desc_type == "vnf":
 935                     get_charm_list = self._charms_search_on_vnfd_sol006_dict
 936                 else:
 937                     raise Exception("Bad descriptor type")
 938                 charms_set = get_charm_list(descriptor_dict)
 939         return charms_set
 940
 941     def _charms_search_on_osm_im_dict(self, osm_im_dict, desc_type):
 942         self._logger.debug("")
 943         charms_set = set()
 944         for k1, v1 in osm_im_dict.items():
 945             for k2, v2 in v1.items():
 946                 for entry in v2:
 947                     if "{}-configuration".format(desc_type) in entry:
 948                         vnf_config = entry["{}-configuration".format(desc_type)]
 949                         for k3, v3 in vnf_config.items():
 950                             if "charm" in v3:
 951                                 charms_set.add((v3["charm"]))
 952                     if "vdu" in entry:
 953                         vdus = entry["vdu"]
 954                         for vdu in vdus:
 955                             if "vdu-configuration" in vdu:
 956                                 for k4, v4 in vdu["vdu-configuration"].items():
 957                                     if "charm" in v4:
 958                                         charms_set.add((v4["charm"]))
 959         return charms_set
 960
 961     def _charms_search_on_vnfd_sol006_dict(self, sol006_dict):
 962         self._logger.debug("")
 963         charms_set = set()
 964         dfs = sol006_dict.get("vnfd", {}).get("df", [])
 965         for df in dfs:
 966             day_1_2s = (
 967                 df.get("lcm-operations-configuration", {})
 968                 .get("operate-vnf-op-config", {})
 969                 .get("day1-2")
 970             )
 971             if day_1_2s is not None:
 972                 for day_1_2 in day_1_2s:
 973                     exec_env_list = day_1_2.get("execution-environment-list", [])
 974                     for exec_env in exec_env_list:
 975                         if "juju" in exec_env and "charm" in exec_env["juju"]:
 976                             charms_set.add(exec_env["juju"]["charm"])
 977         return charms_set
 978
 979     def _charms_search_on_nsd_sol006_dict(self, sol006_dict):
 980         self._logger.debug("")
 981         charms_set = set()
 982         nsd_list = sol006_dict.get("nsd", {}).get("nsd", [])
 983         for nsd in nsd_list:
 984             charm = nsd.get("ns-configuration", {}).get("juju", {}).get("charm")
 985             if charm:
 986                 charms_set.add(charm)
 987         return charms_set