osmclient/common/package_tool.py

   1 # /bin/env python3
   2 # Copyright 2019 ATOS
   3 #
   4 # All Rights Reserved.
   5 #
   6 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 #    not use this file except in compliance with the License. You may obtain
   8 #    a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #    Unless required by applicable law or agreed to in writing, software
  13 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 #    License for the specific language governing permissions and limitations
  16 #    under the License.
  17
  18 import glob
  19 import hashlib
  20 import logging
  21 import os
  22 import shutil
  23 import subprocess
  24 import tarfile
  25 import time
  26
  27 from jinja2 import Environment, PackageLoader
  28 from osm_im.validation import Validation as validation_im
  29 from osm_im.validation import ValidationException
  30 from osm_im import im_translation
  31 from osmclient.common.exceptions import ClientException
  32 import yaml
  33
  34
  35 class PackageTool(object):
  36     def __init__(self, client=None):
  37         self._client = client
  38         self._logger = logging.getLogger("osmclient")
  39         self._validator = validation_im()
  40
  41     def create(
  42         self,
  43         package_type,
  44         base_directory,
  45         package_name,
  46         override,
  47         image,
  48         vdus,
  49         vcpu,
  50         memory,
  51         storage,
  52         interfaces,
  53         vendor,
  54         detailed,
  55         netslice_subnets,
  56         netslice_vlds,
  57         old,
  58     ):
  59         """
  60         **Create a package descriptor**
  61
  62         :params:
  63             - package_type: [vnf, ns, nst]
  64             - base directory: path of destination folder
  65             - package_name: is the name of the package to be created
  66             - image: specify the image of the vdu
  67             - vcpu: number of virtual cpus of the vdu
  68             - memory: amount of memory in MB pf the vdu
  69             - storage: amount of storage in GB of the vdu
  70             - interfaces: number of interfaces besides management interface
  71             - vendor: vendor name of the vnf/ns
  72             - detailed: include all possible values for NSD, VNFD, NST
  73             - netslice_subnets: number of netslice_subnets for the NST
  74             - netslice_vlds: number of virtual link descriptors for the NST
  75             - old: flag to create a descriptor using the previous OSM format (pre SOL006, OSM<9)
  76
  77         :return: status
  78         """
  79         self._logger.debug("")
  80         # print("location: {}".format(osmclient.__path__))
  81         file_loader = PackageLoader("osmclient")
  82         env = Environment(loader=file_loader)
  83         if package_type == "ns":
  84             template = env.get_template("nsd.yaml.j2" if not old else "nsd_old.yaml.j2")
  85             content = {
  86                 "name": package_name,
  87                 "vendor": vendor,
  88                 "vdus": vdus,
  89                 "clean": False,
  90                 "interfaces": interfaces,
  91                 "detailed": detailed,
  92             }
  93         elif package_type == "vnf":
  94             template = env.get_template("vnfd.yaml.j2" if not old else "vnfd_old.yaml.j2")
  95             content = {
  96                 "name": package_name,
  97                 "vendor": vendor,
  98                 "vdus": vdus,
  99                 "clean": False,
 100                 "interfaces": interfaces,
 101                 "image": image,
 102                 "vcpu": vcpu,
 103                 "memory": memory,
 104                 "storage": storage,
 105                 "detailed": detailed,
 106             }
 107         elif package_type == "nst":
 108             # TODO: repo-index did not support nst in OSM<9, no changes in template
 109             template = env.get_template("nst.yaml.j2")
 110             content = {
 111                 "name": package_name,
 112                 "vendor": vendor,
 113                 "interfaces": interfaces,
 114                 "netslice_subnets": netslice_subnets,
 115                 "netslice_vlds": netslice_vlds,
 116                 "detailed": detailed,
 117             }
 118         else:
 119             raise ClientException(
 120                 "Wrong descriptor type {}. Options: ns, vnf, nst".format(package_type)
 121             )
 122
 123         self._logger.debug("To be rendered: {}".format(content))
 124         output = template.render(content)
 125         self._logger.debug(output)
 126
 127         structure = self.discover_folder_structure(
 128             base_directory, package_name, override
 129         )
 130         if structure.get("folders"):
 131             self.create_folders(structure["folders"], package_type)
 132         if structure.get("files"):
 133             self.create_files(structure["files"], output, package_type)
 134         return "Created"
 135
 136     def validate(self, base_directory, recursive=True, old_format=False):
 137         """
 138         **Validate OSM Descriptors given a path**
 139
 140         :params:
 141             - base_directory is the root path for all descriptors
 142
 143         :return: List of dict of validated descriptors. keys: type, path, valid, error
 144         """
 145         self._logger.debug("")
 146         table = []
 147         if recursive:
 148             descriptors_paths = [
 149                 f for f in glob.glob(base_directory + "/**/*.yaml", recursive=recursive)
 150             ]
 151         else:
 152             descriptors_paths = [
 153                 f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)
 154             ]
 155         self._logger.info("Base directory: {}".format(base_directory))
 156         self._logger.info("{} Descriptors found to validate".format(len(descriptors_paths)))
 157         for desc_path in descriptors_paths:
 158             with open(desc_path) as descriptor_file:
 159                 descriptor_data = descriptor_file.read()
 160             desc_type = "-"
 161             try:
 162                 # TODO: refactor validation_im.yaml_validation to @staticmethod
 163                 desc_type, descriptor_data = validation_im.yaml_validation(
 164                     self, descriptor_data
 165                 )
 166                 self._logger.debug(f"Validate {desc_type} {descriptor_data}")
 167                 if not old_format:
 168                     if desc_type == "vnfd" or desc_type == "nsd":
 169                         self._logger.error(
 170                             "OSM descriptor '{}' written in an unsupported format. Please update to ETSI SOL006 format".format(
 171                                 desc_path
 172                             )
 173                         )
 174                         self._logger.warning(
 175                             "Package validation skipped. It can still be done with 'osm package-validate --old'"
 176                         )
 177                         self._logger.warning(
 178                             "Package build can still be done with 'osm package-build --skip-validation'"
 179                         )
 180                         raise Exception("Not SOL006 format")
 181                 validation_im.pyangbind_validation(self, desc_type, descriptor_data)
 182                 table.append(
 183                     {"type": desc_type, "path": desc_path, "valid": "OK", "error": "-"}
 184                 )
 185             except Exception as e:
 186                 self._logger.error(f"Validation error: {e}", exc_info=True)
 187                 table.append(
 188                     {
 189                         "type": desc_type,
 190                         "path": desc_path,
 191                         "valid": "ERROR",
 192                         "error": str(e),
 193                     }
 194                 )
 195             self._logger.debug(table[-1])
 196         return table
 197
 198     def translate(self, base_directory, recursive=True, dryrun=False):
 199         """
 200         **Translate OSM Packages given a path**
 201
 202         :params:
 203             - base_directory is the root path for all packages
 204
 205         :return: List of dict of translated packages. keys: current type, new type, path, valid, translated, error
 206         """
 207         self._logger.debug("")
 208         table = []
 209         if recursive:
 210             descriptors_paths = [
 211                 f for f in glob.glob(base_directory + "/**/*.yaml", recursive=recursive)
 212             ]
 213         else:
 214             descriptors_paths = [
 215                 f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)
 216             ]
 217         print("Base directory: {}".format(base_directory))
 218         print("{} Descriptors found to validate".format(len(descriptors_paths)))
 219         for desc_path in descriptors_paths:
 220             with open(desc_path) as descriptor_file:
 221                 descriptor_data = descriptor_file.read()
 222             desc_type = "-"
 223             try:
 224                 desc_type, descriptor_data = validation_im.yaml_validation(
 225                     self, descriptor_data
 226                 )
 227                 self._logger.debug("desc_type: {}".format(desc_type))
 228                 self._logger.debug("descriptor_data:\n{}".format(descriptor_data))
 229                 self._validator.pyangbind_validation(desc_type, descriptor_data)
 230                 if not (desc_type == "vnfd" or desc_type == "nsd"):
 231                     table.append(
 232                         {
 233                             "current type": desc_type,
 234                             "new type": desc_type,
 235                             "path": desc_path,
 236                             "valid": "OK",
 237                             "translated": "N/A",
 238                             "error": "-",
 239                         }
 240                     )
 241                 else:
 242                     new_desc_type = desc_type
 243                     try:
 244                         sol006_model = yaml.safe_dump(
 245                             im_translation.translate_im_model_to_sol006(
 246                                 descriptor_data
 247                             ),
 248                             indent=4,
 249                             default_flow_style=False,
 250                         )
 251                         (
 252                             new_desc_type,
 253                             new_descriptor_data,
 254                         ) = self._validator.yaml_validation(sol006_model)
 255                         self._validator.pyangbind_validation(
 256                             new_desc_type, new_descriptor_data
 257                         )
 258                         if not dryrun:
 259                             with open(desc_path, "w") as descriptor_file:
 260                                 descriptor_file.write(sol006_model)
 261                         table.append(
 262                             {
 263                                 "current type": desc_type,
 264                                 "new type": new_desc_type,
 265                                 "path": desc_path,
 266                                 "valid": "OK",
 267                                 "translated": "OK",
 268                                 "error": "-",
 269                             }
 270                         )
 271                     except ValidationException as ve2:
 272                         table.append(
 273                             {
 274                                 "current type": desc_type,
 275                                 "new type": new_desc_type,
 276                                 "path": desc_path,
 277                                 "valid": "OK",
 278                                 "translated": "ERROR",
 279                                 "error": "Error in the post-validation: {}".format(
 280                                     str(ve2)
 281                                 ),
 282                             }
 283                         )
 284                     except Exception as e2:
 285                         table.append(
 286                             {
 287                                 "current type": desc_type,
 288                                 "new type": new_desc_type,
 289                                 "path": desc_path,
 290                                 "valid": "OK",
 291                                 "translated": "ERROR",
 292                                 "error": "Error in the translation: {}".format(str(e2)),
 293                             }
 294                         )
 295             except ValidationException as ve:
 296                 table.append(
 297                     {
 298                         "current type": desc_type,
 299                         "new type": "N/A",
 300                         "path": desc_path,
 301                         "valid": "ERROR",
 302                         "translated": "N/A",
 303                         "error": "Error in the pre-validation: {}".format(str(ve)),
 304                     }
 305                 )
 306             except Exception as e:
 307                 table.append(
 308                     {
 309                         "current type": desc_type,
 310                         "new type": "N/A",
 311                         "path": desc_path,
 312                         "valid": "ERROR",
 313                         "translated": "N/A",
 314                         "error": str(e),
 315                     }
 316                 )
 317         return table
 318
 319     def descriptor_translate(self, descriptor_file):
 320         """
 321         **Translate input descriptor file from Rel EIGHT OSM to SOL006**
 322
 323         :params:
 324             - base_directory is the root path for all packages
 325
 326         :return: YAML descriptor in the new format
 327         """
 328         self._logger.debug("")
 329         with open(descriptor_file, "r") as df:
 330             im_model = yaml.safe_load(df.read())
 331         sol006_model = im_translation.translate_im_model_to_sol006(im_model)
 332         return yaml.safe_dump(sol006_model, indent=4, default_flow_style=False)
 333
 334     def build(self, package_folder, skip_validation=False, skip_charm_build=False):
 335         """
 336         **Creates a .tar.gz file given a package_folder**
 337
 338         :params:
 339             - package_folder: is the name of the folder to be packaged
 340             - skip_validation: is the flag to validate or not the descriptors on the folder before build
 341
 342         :returns: message result for the build process
 343         """
 344         self._logger.debug("")
 345         package_folder = package_folder.rstrip("/")
 346         if not os.path.exists("{}".format(package_folder)):
 347             return "Fail, package is not in the specified path"
 348         if not skip_validation:
 349             print("Validating package {}".format(package_folder))
 350             results = self.validate(package_folder, recursive=False)
 351             if results:
 352                 for result in results:
 353                     if result["valid"] != "OK":
 354                         raise ClientException(
 355                             "There was an error validating the file {} with error: {}".format(
 356                                 result["path"], result["error"]
 357                             )
 358                         )
 359                 print("Validation OK")
 360             else:
 361                 raise ClientException(
 362                     "No descriptor file found in: {}".format(package_folder)
 363                 )
 364         charm_list = self.build_all_charms(package_folder, skip_charm_build)
 365         return self.build_tarfile(package_folder, charm_list)
 366
 367     def calculate_checksum(self, package_folder):
 368         """
 369         **Function to calculate the checksum given a folder**
 370
 371         :params:
 372             - package_folder: is the folder where we have the files to calculate the checksum
 373         :returns: None
 374         """
 375         self._logger.debug("")
 376         files = [
 377             f
 378             for f in glob.glob(package_folder + "/**/*.*", recursive=True)
 379             if os.path.isfile(f)
 380         ]
 381         with open("{}/checksums.txt".format(package_folder), "w+") as checksum:
 382             for file_item in files:
 383                 if "checksums.txt" in file_item:
 384                     continue
 385                 # from https://www.quickprogrammingtips.com/python/how-to-calculate-md5-hash-of-a-file-in-python.html
 386                 md5_hash = hashlib.md5()
 387                 with open(file_item, "rb") as f:
 388                     # Read and update hash in chunks of 4K
 389                     for byte_block in iter(lambda: f.read(4096), b""):
 390                         md5_hash.update(byte_block)
 391                     checksum.write("{}\t{}\n".format(md5_hash.hexdigest(), file_item))
 392
 393     def create_folders(self, folders, package_type):
 394         """
 395         **Create folder given a list of folders**
 396
 397         :params:
 398             - folders: [List] list of folders paths to be created
 399             - package_type: is the type of package to be created
 400         :return: None
 401         """
 402         self._logger.debug("")
 403         for folder in folders:
 404             try:
 405                 # print("Folder {} == package_type {}".format(folder[1], package_type))
 406                 if folder[1] == package_type:
 407                     print("Creating folder:\t{}".format(folder[0]))
 408                     os.makedirs(folder[0])
 409             except FileExistsError:
 410                 pass
 411
 412     def save_file(self, file_name, file_body):
 413         """
 414         **Create a file given a name and the content**
 415
 416         :params:
 417             - file_name: is the name of the file with the relative route
 418             - file_body: is the content of the file
 419         :return: None
 420         """
 421         self._logger.debug("")
 422         print("Creating file:  \t{}".format(file_name))
 423         try:
 424             with open(file_name, "w+") as f:
 425                 f.write(file_body)
 426         except Exception as e:
 427             raise ClientException(e)
 428
 429     def generate_readme(self):
 430         """
 431         **Creates the README content**
 432
 433         :returns: readme content
 434         """
 435         self._logger.debug("")
 436         return """# Descriptor created by OSM descriptor package generated\n\n**Created on {} **""".format(
 437             time.strftime("%m/%d/%Y, %H:%M:%S", time.localtime())
 438         )
 439
 440     def generate_cloud_init(self):
 441         """
 442         **Creates the cloud-init content**
 443
 444         :returns: cloud-init content
 445         """
 446         self._logger.debug("")
 447         return "---\n#cloud-config"
 448
 449     def create_files(self, files, file_content, package_type):
 450         """
 451         **Creates the files given the file list and type**
 452
 453         :params:
 454             - files: is the list of files structure
 455             - file_content: is the content of the descriptor rendered by the template
 456             - package_type: is the type of package to filter the creation structure
 457
 458         :return: None
 459         """
 460         self._logger.debug("")
 461         for file_item, file_package, file_type in files:
 462             if package_type == file_package:
 463                 if file_type == "descriptor":
 464                     self.save_file(file_item, file_content)
 465                 elif file_type == "readme":
 466                     self.save_file(file_item, self.generate_readme())
 467                 elif file_type == "cloud_init":
 468                     self.save_file(file_item, self.generate_cloud_init())
 469
 470     def check_files_folders(self, path_list, override):
 471         """
 472         **Find files and folders missing given a directory structure {"folders": [], "files": []}**
 473
 474         :params:
 475             - path_list: is the list of files and folders to be created
 476             - override: is the flag used to indicate the creation of the list even if the file exist to override it
 477
 478         :return: Missing paths Dict
 479         """
 480         self._logger.debug("")
 481         missing_paths = {}
 482         folders = []
 483         files = []
 484         for folder in path_list.get("folders"):
 485             if not os.path.exists(folder[0]):
 486                 folders.append(folder)
 487         missing_paths["folders"] = folders
 488
 489         for file_item in path_list.get("files"):
 490             if not os.path.exists(file_item[0]) or override is True:
 491                 files.append(file_item)
 492         missing_paths["files"] = files
 493
 494         return missing_paths
 495
 496     def build_all_charms(self, package_folder, skip_charm_build):
 497         """
 498         **Read the descriptor file, check that the charms referenced are in the folder and compiles them**
 499
 500         :params:
 501             - packet_folder: is the location of the package
 502         :return: Files and Folders not found. In case of override, it will return all file list
 503         """
 504         self._logger.debug("")
 505         charms_set = set()
 506         descriptor_file = False
 507         descriptors_paths = [f for f in glob.glob(package_folder + "/*.yaml")]
 508         for file in descriptors_paths:
 509             if file.endswith("nfd.yaml"):
 510                 descriptor_file = True
 511                 charms_set = self.charms_search(file, "vnf")
 512             if file.endswith("nsd.yaml"):
 513                 descriptor_file = True
 514                 charms_set = self.charms_search(file, "ns")
 515         print("List of charms in the descriptor: {}".format(charms_set))
 516         if not descriptor_file:
 517             raise ClientException(
 518                 'Descriptor filename is not correct in: {}. It should end with "nfd.yaml" or "nsd.yaml"'.format(
 519                     package_folder
 520                 )
 521             )
 522         if charms_set and not skip_charm_build:
 523             for charmName in charms_set:
 524                 if os.path.isdir(
 525                     "{}/charms/layers/{}".format(package_folder, charmName)
 526                 ):
 527                     print(
 528                         "Building charm {}/charms/layers/{}".format(
 529                             package_folder, charmName
 530                         )
 531                     )
 532                     self.charm_build(package_folder, charmName)
 533                     print("Charm built: {}".format(charmName))
 534                 elif os.path.isdir(
 535                     "{}/charms/ops/{}".format(package_folder, charmName)
 536                 ):
 537                     self.charmcraft_build(package_folder, charmName)
 538                 else:
 539                     if not os.path.isdir(
 540                         "{}/charms/{}".format(package_folder, charmName)
 541                     ):
 542                         raise ClientException(
 543                             "The charm: {} referenced in the descriptor file "
 544                             "is not present either in {}/charms or in {}/charms/layers".format(
 545                                 charmName, package_folder, package_folder
 546                             )
 547                         )
 548         self._logger.debug("Return list of charms: {}".format(charms_set))
 549         return charms_set
 550
 551     def discover_folder_structure(self, base_directory, name, override):
 552         """
 553         **Discover files and folders structure for OSM descriptors given a base_directory and name**
 554
 555         :params:
 556             - base_directory: is the location of the package to be created
 557             - name: is the name of the package
 558             - override: is the flag used to indicate the creation of the list even if the file exist to override it
 559         :return: Files and Folders not found. In case of override, it will return all file list
 560         """
 561         self._logger.debug("")
 562         prefix = "{}/{}".format(base_directory, name)
 563         files_folders = {
 564             "folders": [
 565                 ("{}_ns".format(prefix), "ns"),
 566                 ("{}_ns/icons".format(prefix), "ns"),
 567                 ("{}_ns/charms".format(prefix), "ns"),
 568                 ("{}_vnf".format(name), "vnf"),
 569                 ("{}_vnf/charms".format(prefix), "vnf"),
 570                 ("{}_vnf/cloud_init".format(prefix), "vnf"),
 571                 ("{}_vnf/images".format(prefix), "vnf"),
 572                 ("{}_vnf/icons".format(prefix), "vnf"),
 573                 ("{}_vnf/scripts".format(prefix), "vnf"),
 574                 ("{}_nst".format(prefix), "nst"),
 575                 ("{}_nst/icons".format(prefix), "nst"),
 576             ],
 577             "files": [
 578                 ("{}_ns/{}_nsd.yaml".format(prefix, name), "ns", "descriptor"),
 579                 ("{}_ns/README.md".format(prefix), "ns", "readme"),
 580                 ("{}_vnf/{}_vnfd.yaml".format(prefix, name), "vnf", "descriptor"),
 581                 (
 582                     "{}_vnf/cloud_init/cloud-config.txt".format(prefix),
 583                     "vnf",
 584                     "cloud_init",
 585                 ),
 586                 ("{}_vnf/README.md".format(prefix), "vnf", "readme"),
 587                 ("{}_nst/{}_nst.yaml".format(prefix, name), "nst", "descriptor"),
 588                 ("{}_nst/README.md".format(prefix), "nst", "readme"),
 589             ],
 590         }
 591         missing_files_folders = self.check_files_folders(files_folders, override)
 592         # print("Missing files and folders: {}".format(missing_files_folders))
 593         return missing_files_folders
 594
 595     def charm_build(self, charms_folder, build_name):
 596         """
 597         Build the charms inside the package.
 598         params: package_folder is the name of the folder where is the charms to compile.
 599                 build_name is the name of the layer or interface
 600         """
 601         self._logger.debug("")
 602         os.environ["JUJU_REPOSITORY"] = "{}/charms".format(charms_folder)
 603         os.environ["CHARM_LAYERS_DIR"] = "{}/layers".format(
 604             os.environ["JUJU_REPOSITORY"]
 605         )
 606         os.environ["CHARM_INTERFACES_DIR"] = "{}/interfaces".format(
 607             os.environ["JUJU_REPOSITORY"]
 608         )
 609         os.environ["CHARM_BUILD_DIR"] = "{}/charms/builds".format(charms_folder)
 610         if not os.path.exists(os.environ["CHARM_BUILD_DIR"]):
 611             os.makedirs(os.environ["CHARM_BUILD_DIR"])
 612         src_folder = "{}/{}".format(os.environ["CHARM_LAYERS_DIR"], build_name)
 613         result = subprocess.run(["charm", "build", "{}".format(src_folder)])
 614         if result.returncode == 1:
 615             raise ClientException("failed to build the charm: {}".format(src_folder))
 616         self._logger.verbose("charm {} built".format(src_folder))
 617
 618     def charmcraft_build(self, package_folder, charm_name):
 619         """
 620         Build the charms inside the package (new operator framework charms)
 621         params: package_folder is the name of the folder where is the charms to compile.
 622                 build_name is the name of the layer or interface
 623         """
 624         self._logger.debug("Building charm {}".format(charm_name))
 625         src_folder = f"{package_folder}/charms/ops/{charm_name}"
 626         current_directory = os.getcwd()
 627         os.chdir(src_folder)
 628         try:
 629             result = subprocess.run(["charmcraft", "build"])
 630             if result.returncode == 1:
 631                 raise ClientException(
 632                     "failed to build the charm: {}".format(src_folder)
 633                 )
 634             subprocess.run(["rm", "-rf", f"../../{charm_name}"])
 635             subprocess.run(["mv", "build", f"../../{charm_name}"])
 636             self._logger.verbose("charm {} built".format(src_folder))
 637         finally:
 638             os.chdir(current_directory)
 639
 640     def build_tarfile(self, package_folder, charm_list=None):
 641         """
 642         Creates a .tar.gz file given a package_folder
 643         params: package_folder is the name of the folder to be packaged
 644         returns: .tar.gz name
 645         """
 646         self._logger.debug("")
 647         cwd = None
 648         try:
 649             directory_name, package_name = self.create_temp_dir(
 650                 package_folder, charm_list
 651             )
 652             cwd = os.getcwd()
 653             os.chdir(directory_name)
 654             self.calculate_checksum(package_name)
 655             with tarfile.open("{}.tar.gz".format(package_name), mode="w:gz") as archive:
 656                 print("Adding File: {}".format(package_name))
 657                 archive.add("{}".format(package_name), recursive=True)
 658             # return "Created {}.tar.gz".format(package_folder)
 659             # self.build("{}".format(os.path.basename(package_folder)))
 660             os.chdir(cwd)
 661             cwd = None
 662             created_package = "{}/{}.tar.gz".format(
 663                 os.path.dirname(package_folder) or ".", package_name
 664             )
 665             os.rename(
 666                 "{}/{}.tar.gz".format(directory_name, package_name), created_package
 667             )
 668             os.rename(
 669                 "{}/{}/checksums.txt".format(directory_name, package_name),
 670                 "{}/checksums.txt".format(package_folder),
 671             )
 672             print("Package created: {}".format(created_package))
 673             return created_package
 674         except Exception as exc:
 675             raise ClientException(
 676                 "failure during build of targz file (create temp dir, calculate checksum, "
 677                 "tar.gz file): {}".format(exc)
 678             )
 679         finally:
 680             if cwd:
 681                 os.chdir(cwd)
 682             shutil.rmtree(os.path.join(package_folder, "tmp"))
 683
 684     def create_temp_dir(self, package_folder, charm_list=None):
 685         """
 686         Method to create a temporary folder where we can move the files in package_folder
 687         """
 688         self._logger.debug("")
 689         ignore_patterns = ".gitignore"
 690         ignore = shutil.ignore_patterns(ignore_patterns)
 691         directory_name = os.path.abspath(package_folder)
 692         package_name = os.path.basename(directory_name)
 693         directory_name += "/tmp"
 694         os.makedirs("{}/{}".format(directory_name, package_name), exist_ok=True)
 695         self._logger.debug("Makedirs DONE: {}/{}".format(directory_name, package_name))
 696         for item in os.listdir(package_folder):
 697             self._logger.debug("Item: {}".format(item))
 698             if item != "tmp":
 699                 s = os.path.join(package_folder, item)
 700                 d = os.path.join(os.path.join(directory_name, package_name), item)
 701                 if os.path.isdir(s):
 702                     if item == "charms":
 703                         os.makedirs(d, exist_ok=True)
 704                         s_builds = os.path.join(s, "builds")
 705                         for charm in charm_list:
 706                             self._logger.debug("Copying charm {}".format(charm))
 707                             if charm in os.listdir(s):
 708                                 s_charm = os.path.join(s, charm)
 709                             elif charm in os.listdir(s_builds):
 710                                 s_charm = os.path.join(s_builds, charm)
 711                             else:
 712                                 raise ClientException(
 713                                     "The charm {} referenced in the descriptor file "
 714                                     "could not be found in {}/charms or in {}/charms/builds".format(
 715                                         charm, package_folder, package_folder
 716                                     )
 717                                 )
 718                             d_temp = os.path.join(d, charm)
 719                             self._logger.debug(
 720                                 "Copying tree: {} -> {}".format(s_charm, d_temp)
 721                             )
 722                             shutil.copytree(
 723                                 s_charm, d_temp, symlinks=True, ignore=ignore
 724                             )
 725                             self._logger.debug("DONE")
 726                     else:
 727                         self._logger.debug("Copying tree: {} -> {}".format(s, d))
 728                         shutil.copytree(s, d, symlinks=True, ignore=ignore)
 729                         self._logger.debug("DONE")
 730                 else:
 731                     if item in ignore_patterns:
 732                         continue
 733                     self._logger.debug("Copying file: {} -> {}".format(s, d))
 734                     shutil.copy2(s, d)
 735                     self._logger.debug("DONE")
 736         return directory_name, package_name
 737
 738     def charms_search(self, descriptor_file, desc_type):
 739         self._logger.debug(
 740             "descriptor_file: {}, desc_type: {}".format(descriptor_file, desc_type)
 741         )
 742         charms_set = set()
 743         with open("{}".format(descriptor_file)) as yaml_desc:
 744             descriptor_dict = yaml.safe_load(yaml_desc)
 745             # self._logger.debug("\n"+yaml.safe_dump(descriptor_dict, indent=4, default_flow_style=False))
 746
 747             if (
 748                 desc_type == "vnf"
 749                 and (
 750                     "vnfd:vnfd-catalog" in descriptor_dict
 751                     or "vnfd-catalog" in descriptor_dict
 752                 )
 753             ) or (
 754                 desc_type == "ns"
 755                 and (
 756                     "nsd:nsd-catalog" in descriptor_dict
 757                     or "nsd-catalog" in descriptor_dict
 758                 )
 759             ):
 760                 charms_set = self._charms_search_on_osm_im_dict(
 761                     descriptor_dict, desc_type
 762                 )
 763             else:
 764                 if desc_type == "ns":
 765                     get_charm_list = self._charms_search_on_nsd_sol006_dict
 766                 elif desc_type == "vnf":
 767                     get_charm_list = self._charms_search_on_vnfd_sol006_dict
 768                 else:
 769                     raise Exception("Bad descriptor type")
 770                 charms_set = get_charm_list(descriptor_dict)
 771         return charms_set
 772
 773     def _charms_search_on_osm_im_dict(self, osm_im_dict, desc_type):
 774         self._logger.debug("")
 775         charms_set = []
 776         for k1, v1 in osm_im_dict.items():
 777             for k2, v2 in v1.items():
 778                 for entry in v2:
 779                     if "{}-configuration".format(desc_type) in entry:
 780                         vnf_config = entry["{}-configuration".format(desc_type)]
 781                         for k3, v3 in vnf_config.items():
 782                             if "charm" in v3:
 783                                 charms_set.add((v3["charm"]))
 784                     if "vdu" in entry:
 785                         vdus = entry["vdu"]
 786                         for vdu in vdus:
 787                             if "vdu-configuration" in vdu:
 788                                 for k4, v4 in vdu["vdu-configuration"].items():
 789                                     if "charm" in v4:
 790                                         charms_set.add((v4["charm"]))
 791         return charms_set
 792
 793     def _charms_search_on_vnfd_sol006_dict(self, sol006_dict):
 794         self._logger.debug("")
 795         charms_set = set()
 796         dfs = sol006_dict.get("vnfd", {}).get("df", [])
 797         for df in dfs:
 798             day_1_2s = (
 799                 df.get("lcm-operations-configuration", {})
 800                 .get("operate-vnf-op-config", {})
 801                 .get("day1-2")
 802             )
 803             if day_1_2s is not None:
 804                 for day_1_2 in day_1_2s:
 805                     exec_env_list = day_1_2.get("execution-environment-list", [])
 806                     for exec_env in exec_env_list:
 807                         if "juju" in exec_env and "charm" in exec_env["juju"]:
 808                             charms_set.add(exec_env["juju"]["charm"])
 809         return charms_set
 810
 811     def _charms_search_on_nsd_sol006_dict(self, sol006_dict):
 812         self._logger.debug("")
 813         charms_set = set()
 814         nsd_list = sol006_dict.get("nsd", {}).get("nsd", [])
 815         for nsd in nsd_list:
 816             charm = nsd.get("ns-configuration", {}).get("juju", {}).get("charm")
 817             if charm:
 818                 charms_set.add(charm)
 819         return charms_set