osmclient/common/package_tool.py

   1 # /bin/env python3
   2 # Copyright 2019 ATOS
   3 #
   4 # All Rights Reserved.
   5 #
   6 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 #    not use this file except in compliance with the License. You may obtain
   8 #    a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #    Unless required by applicable law or agreed to in writing, software
  13 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 #    License for the specific language governing permissions and limitations
  16 #    under the License.
  17
  18 import glob
  19 import hashlib
  20 import logging
  21 import os
  22 import shutil
  23 import subprocess
  24 import tarfile
  25 import time
  26
  27 from jinja2 import Environment, PackageLoader
  28 from osm_im.validation import Validation as validation_im
  29 from osm_im.validation import ValidationException
  30 from osm_im import im_translation
  31 from osmclient.common.exceptions import ClientException
  32 import yaml
  33
  34
  35 class PackageTool(object):
  36     def __init__(self, client=None):
  37         self._client = client
  38         self._logger = logging.getLogger("osmclient")
  39         self._validator = validation_im()
  40
  41     def create(
  42         self,
  43         package_type,
  44         base_directory,
  45         package_name,
  46         override,
  47         image,
  48         vdus,
  49         vcpu,
  50         memory,
  51         storage,
  52         interfaces,
  53         vendor,
  54         detailed,
  55         netslice_subnets,
  56         netslice_vlds,
  57     ):
  58         """
  59         **Create a package descriptor**
  60
  61         :params:
  62             - package_type: [vnf, ns, nst]
  63             - base directory: path of destination folder
  64             - package_name: is the name of the package to be created
  65             - image: specify the image of the vdu
  66             - vcpu: number of virtual cpus of the vdu
  67             - memory: amount of memory in MB pf the vdu
  68             - storage: amount of storage in GB of the vdu
  69             - interfaces: number of interfaces besides management interface
  70             - vendor: vendor name of the vnf/ns
  71             - detailed: include all possible values for NSD, VNFD, NST
  72             - netslice_subnets: number of netslice_subnets for the NST
  73             - netslice_vlds: number of virtual link descriptors for the NST
  74
  75         :return: status
  76         """
  77         self._logger.debug("")
  78         # print("location: {}".format(osmclient.__path__))
  79         file_loader = PackageLoader("osmclient")
  80         env = Environment(loader=file_loader)
  81         if package_type == "ns":
  82             template = env.get_template("nsd.yaml.j2")
  83             content = {
  84                 "name": package_name,
  85                 "vendor": vendor,
  86                 "vdus": vdus,
  87                 "clean": False,
  88                 "interfaces": interfaces,
  89                 "detailed": detailed,
  90             }
  91         elif package_type == "vnf":
  92             template = env.get_template("vnfd.yaml.j2")
  93             content = {
  94                 "name": package_name,
  95                 "vendor": vendor,
  96                 "vdus": vdus,
  97                 "clean": False,
  98                 "interfaces": interfaces,
  99                 "image": image,
 100                 "vcpu": vcpu,
 101                 "memory": memory,
 102                 "storage": storage,
 103                 "detailed": detailed,
 104             }
 105         elif package_type == "nst":
 106             template = env.get_template("nst.yaml.j2")
 107             content = {
 108                 "name": package_name,
 109                 "vendor": vendor,
 110                 "interfaces": interfaces,
 111                 "netslice_subnets": netslice_subnets,
 112                 "netslice_vlds": netslice_vlds,
 113                 "detailed": detailed,
 114             }
 115         else:
 116             raise ClientException(
 117                 "Wrong descriptor type {}. Options: ns, vnf, nst".format(package_type)
 118             )
 119
 120         # print("To be rendered: {}".format(content))
 121         output = template.render(content)
 122         # print(output)
 123
 124         structure = self.discover_folder_structure(
 125             base_directory, package_name, override
 126         )
 127         if structure.get("folders"):
 128             self.create_folders(structure["folders"], package_type)
 129         if structure.get("files"):
 130             self.create_files(structure["files"], output, package_type)
 131         return "Created"
 132
 133     def validate(self, base_directory, recursive=True, old_format=False):
 134         """
 135         **Validate OSM Descriptors given a path**
 136
 137         :params:
 138             - base_directory is the root path for all descriptors
 139
 140         :return: List of dict of validated descriptors. keys: type, path, valid, error
 141         """
 142         self._logger.debug("")
 143         table = []
 144         if recursive:
 145             descriptors_paths = [
 146                 f for f in glob.glob(base_directory + "/**/*.yaml", recursive=recursive)
 147             ]
 148         else:
 149             descriptors_paths = [
 150                 f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)
 151             ]
 152         print("Base directory: {}".format(base_directory))
 153         print("{} Descriptors found to validate".format(len(descriptors_paths)))
 154         for desc_path in descriptors_paths:
 155             with open(desc_path) as descriptor_file:
 156                 descriptor_data = descriptor_file.read()
 157             desc_type = "-"
 158             try:
 159                 desc_type, descriptor_data = validation_im.yaml_validation(
 160                     self, descriptor_data
 161                 )
 162                 if not old_format:
 163                     if desc_type == "vnfd" or desc_type == "nsd":
 164                         print(
 165                             "OSM descriptor '{}' written in an unsupported format. Please update to ETSI SOL006 format".format(
 166                                 desc_path
 167                             )
 168                         )
 169                         print(
 170                             "Package validation skipped. It can still be done with 'osm package-validate --old'"
 171                         )
 172                         print(
 173                             "Package build can still be done with 'osm package-build --skip-validation'"
 174                         )
 175                         raise Exception("Not SOL006 format")
 176                 validation_im.pyangbind_validation(self, desc_type, descriptor_data)
 177                 table.append(
 178                     {"type": desc_type, "path": desc_path, "valid": "OK", "error": "-"}
 179                 )
 180             except Exception as e:
 181                 table.append(
 182                     {
 183                         "type": desc_type,
 184                         "path": desc_path,
 185                         "valid": "ERROR",
 186                         "error": str(e),
 187                     }
 188                 )
 189         return table
 190
 191     def translate(self, base_directory, recursive=True, dryrun=False):
 192         """
 193         **Translate OSM Packages given a path**
 194
 195         :params:
 196             - base_directory is the root path for all packages
 197
 198         :return: List of dict of translated packages. keys: current type, new type, path, valid, translated, error
 199         """
 200         self._logger.debug("")
 201         table = []
 202         if recursive:
 203             descriptors_paths = [
 204                 f for f in glob.glob(base_directory + "/**/*.yaml", recursive=recursive)
 205             ]
 206         else:
 207             descriptors_paths = [
 208                 f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)
 209             ]
 210         print("Base directory: {}".format(base_directory))
 211         print("{} Descriptors found to validate".format(len(descriptors_paths)))
 212         for desc_path in descriptors_paths:
 213             with open(desc_path) as descriptor_file:
 214                 descriptor_data = descriptor_file.read()
 215             desc_type = "-"
 216             try:
 217                 desc_type, descriptor_data = validation_im.yaml_validation(
 218                     self, descriptor_data
 219                 )
 220                 self._logger.debug("desc_type: {}".format(desc_type))
 221                 self._logger.debug("descriptor_data:\n{}".format(descriptor_data))
 222                 self._validator.pyangbind_validation(desc_type, descriptor_data)
 223                 if not (desc_type == "vnfd" or desc_type == "nsd"):
 224                     table.append(
 225                         {
 226                             "current type": desc_type,
 227                             "new type": desc_type,
 228                             "path": desc_path,
 229                             "valid": "OK",
 230                             "translated": "N/A",
 231                             "error": "-",
 232                         }
 233                     )
 234                 else:
 235                     new_desc_type = desc_type
 236                     try:
 237                         sol006_model = yaml.safe_dump(
 238                             im_translation.translate_im_model_to_sol006(
 239                                 descriptor_data
 240                             ),
 241                             indent=4,
 242                             default_flow_style=False,
 243                         )
 244                         (
 245                             new_desc_type,
 246                             new_descriptor_data,
 247                         ) = self._validator.yaml_validation(sol006_model)
 248                         self._validator.pyangbind_validation(
 249                             new_desc_type, new_descriptor_data
 250                         )
 251                         if not dryrun:
 252                             with open(desc_path, "w") as descriptor_file:
 253                                 descriptor_file.write(sol006_model)
 254                         table.append(
 255                             {
 256                                 "current type": desc_type,
 257                                 "new type": new_desc_type,
 258                                 "path": desc_path,
 259                                 "valid": "OK",
 260                                 "translated": "OK",
 261                                 "error": "-",
 262                             }
 263                         )
 264                     except ValidationException as ve2:
 265                         table.append(
 266                             {
 267                                 "current type": desc_type,
 268                                 "new type": new_desc_type,
 269                                 "path": desc_path,
 270                                 "valid": "OK",
 271                                 "translated": "ERROR",
 272                                 "error": "Error in the post-validation: {}".format(
 273                                     str(ve2)
 274                                 ),
 275                             }
 276                         )
 277                     except Exception as e2:
 278                         table.append(
 279                             {
 280                                 "current type": desc_type,
 281                                 "new type": new_desc_type,
 282                                 "path": desc_path,
 283                                 "valid": "OK",
 284                                 "translated": "ERROR",
 285                                 "error": "Error in the translation: {}".format(str(e2)),
 286                             }
 287                         )
 288             except ValidationException as ve:
 289                 table.append(
 290                     {
 291                         "current type": desc_type,
 292                         "new type": "N/A",
 293                         "path": desc_path,
 294                         "valid": "ERROR",
 295                         "translated": "N/A",
 296                         "error": "Error in the pre-validation: {}".format(str(ve)),
 297                     }
 298                 )
 299             except Exception as e:
 300                 table.append(
 301                     {
 302                         "current type": desc_type,
 303                         "new type": "N/A",
 304                         "path": desc_path,
 305                         "valid": "ERROR",
 306                         "translated": "N/A",
 307                         "error": str(e),
 308                     }
 309                 )
 310         return table
 311
 312     def descriptor_translate(self, descriptor_file):
 313         """
 314         **Translate input descriptor file from Rel EIGHT OSM to SOL006**
 315
 316         :params:
 317             - base_directory is the root path for all packages
 318
 319         :return: YAML descriptor in the new format
 320         """
 321         self._logger.debug("")
 322         with open(descriptor_file, "r") as df:
 323             im_model = yaml.safe_load(df.read())
 324         sol006_model = im_translation.translate_im_model_to_sol006(im_model)
 325         return yaml.safe_dump(sol006_model, indent=4, default_flow_style=False)
 326
 327     def build(self, package_folder, skip_validation=False, skip_charm_build=False):
 328         """
 329         **Creates a .tar.gz file given a package_folder**
 330
 331         :params:
 332             - package_folder: is the name of the folder to be packaged
 333             - skip_validation: is the flag to validate or not the descriptors on the folder before build
 334
 335         :returns: message result for the build process
 336         """
 337         self._logger.debug("")
 338         package_folder = package_folder.rstrip("/")
 339         if not os.path.exists("{}".format(package_folder)):
 340             return "Fail, package is not in the specified path"
 341         if not skip_validation:
 342             print("Validating package {}".format(package_folder))
 343             results = self.validate(package_folder, recursive=False)
 344             if results:
 345                 for result in results:
 346                     if result["valid"] != "OK":
 347                         raise ClientException(
 348                             "There was an error validating the file {} with error: {}".format(
 349                                 result["path"], result["error"]
 350                             )
 351                         )
 352                 print("Validation OK")
 353             else:
 354                 raise ClientException(
 355                     "No descriptor file found in: {}".format(package_folder)
 356                 )
 357         charm_list = self.build_all_charms(package_folder, skip_charm_build)
 358         return self.build_tarfile(package_folder, charm_list)
 359
 360     def calculate_checksum(self, package_folder):
 361         """
 362         **Function to calculate the checksum given a folder**
 363
 364         :params:
 365             - package_folder: is the folder where we have the files to calculate the checksum
 366         :returns: None
 367         """
 368         self._logger.debug("")
 369         files = [
 370             f
 371             for f in glob.glob(package_folder + "/**/*.*", recursive=True)
 372             if os.path.isfile(f)
 373         ]
 374         with open("{}/checksums.txt".format(package_folder), "w+") as checksum:
 375             for file_item in files:
 376                 if "checksums.txt" in file_item:
 377                     continue
 378                 # from https://www.quickprogrammingtips.com/python/how-to-calculate-md5-hash-of-a-file-in-python.html
 379                 md5_hash = hashlib.md5()
 380                 with open(file_item, "rb") as f:
 381                     # Read and update hash in chunks of 4K
 382                     for byte_block in iter(lambda: f.read(4096), b""):
 383                         md5_hash.update(byte_block)
 384                     checksum.write("{}\t{}\n".format(md5_hash.hexdigest(), file_item))
 385
 386     def create_folders(self, folders, package_type):
 387         """
 388         **Create folder given a list of folders**
 389
 390         :params:
 391             - folders: [List] list of folders paths to be created
 392             - package_type: is the type of package to be created
 393         :return: None
 394         """
 395         self._logger.debug("")
 396         for folder in folders:
 397             try:
 398                 # print("Folder {} == package_type {}".format(folder[1], package_type))
 399                 if folder[1] == package_type:
 400                     print("Creating folder:\t{}".format(folder[0]))
 401                     os.makedirs(folder[0])
 402             except FileExistsError:
 403                 pass
 404
 405     def save_file(self, file_name, file_body):
 406         """
 407         **Create a file given a name and the content**
 408
 409         :params:
 410             - file_name: is the name of the file with the relative route
 411             - file_body: is the content of the file
 412         :return: None
 413         """
 414         self._logger.debug("")
 415         print("Creating file:  \t{}".format(file_name))
 416         try:
 417             with open(file_name, "w+") as f:
 418                 f.write(file_body)
 419         except Exception as e:
 420             raise ClientException(e)
 421
 422     def generate_readme(self):
 423         """
 424         **Creates the README content**
 425
 426         :returns: readme content
 427         """
 428         self._logger.debug("")
 429         return """# Descriptor created by OSM descriptor package generated\n\n**Created on {} **""".format(
 430             time.strftime("%m/%d/%Y, %H:%M:%S", time.localtime())
 431         )
 432
 433     def generate_cloud_init(self):
 434         """
 435         **Creates the cloud-init content**
 436
 437         :returns: cloud-init content
 438         """
 439         self._logger.debug("")
 440         return "---\n#cloud-config"
 441
 442     def create_files(self, files, file_content, package_type):
 443         """
 444         **Creates the files given the file list and type**
 445
 446         :params:
 447             - files: is the list of files structure
 448             - file_content: is the content of the descriptor rendered by the template
 449             - package_type: is the type of package to filter the creation structure
 450
 451         :return: None
 452         """
 453         self._logger.debug("")
 454         for file_item, file_package, file_type in files:
 455             if package_type == file_package:
 456                 if file_type == "descriptor":
 457                     self.save_file(file_item, file_content)
 458                 elif file_type == "readme":
 459                     self.save_file(file_item, self.generate_readme())
 460                 elif file_type == "cloud_init":
 461                     self.save_file(file_item, self.generate_cloud_init())
 462
 463     def check_files_folders(self, path_list, override):
 464         """
 465         **Find files and folders missing given a directory structure {"folders": [], "files": []}**
 466
 467         :params:
 468             - path_list: is the list of files and folders to be created
 469             - override: is the flag used to indicate the creation of the list even if the file exist to override it
 470
 471         :return: Missing paths Dict
 472         """
 473         self._logger.debug("")
 474         missing_paths = {}
 475         folders = []
 476         files = []
 477         for folder in path_list.get("folders"):
 478             if not os.path.exists(folder[0]):
 479                 folders.append(folder)
 480         missing_paths["folders"] = folders
 481
 482         for file_item in path_list.get("files"):
 483             if not os.path.exists(file_item[0]) or override is True:
 484                 files.append(file_item)
 485         missing_paths["files"] = files
 486
 487         return missing_paths
 488
 489     def build_all_charms(self, package_folder, skip_charm_build):
 490         """
 491         **Read the descriptor file, check that the charms referenced are in the folder and compiles them**
 492
 493         :params:
 494             - packet_folder: is the location of the package
 495         :return: Files and Folders not found. In case of override, it will return all file list
 496         """
 497         self._logger.debug("")
 498         charms_set = set()
 499         descriptor_file = False
 500         descriptors_paths = [f for f in glob.glob(package_folder + "/*.yaml")]
 501         for file in descriptors_paths:
 502             if file.endswith("nfd.yaml"):
 503                 descriptor_file = True
 504                 charms_set = self.charms_search(file, "vnf")
 505             if file.endswith("nsd.yaml"):
 506                 descriptor_file = True
 507                 charms_set = self.charms_search(file, "ns")
 508         print("List of charms in the descriptor: {}".format(charms_set))
 509         if not descriptor_file:
 510             raise ClientException(
 511                 'Descriptor filename is not correct in: {}. It should end with "nfd.yaml" or "nsd.yaml"'.format(
 512                     package_folder
 513                 )
 514             )
 515         if charms_set and not skip_charm_build:
 516             for charmName in charms_set:
 517                 if os.path.isdir(
 518                     "{}/charms/layers/{}".format(package_folder, charmName)
 519                 ):
 520                     print(
 521                         "Building charm {}/charms/layers/{}".format(
 522                             package_folder, charmName
 523                         )
 524                     )
 525                     self.charm_build(package_folder, charmName)
 526                     print("Charm built: {}".format(charmName))
 527                 elif os.path.isdir(
 528                     "{}/charms/ops/{}".format(package_folder, charmName)
 529                 ):
 530                     self.charmcraft_build(package_folder, charmName)
 531                 else:
 532                     if not os.path.isdir(
 533                         "{}/charms/{}".format(package_folder, charmName)
 534                     ):
 535                         raise ClientException(
 536                             "The charm: {} referenced in the descriptor file "
 537                             "is not present either in {}/charms or in {}/charms/layers".format(
 538                                 charmName, package_folder, package_folder
 539                             )
 540                         )
 541         self._logger.debug("Return list of charms: {}".format(charms_set))
 542         return charms_set
 543
 544     def discover_folder_structure(self, base_directory, name, override):
 545         """
 546         **Discover files and folders structure for OSM descriptors given a base_directory and name**
 547
 548         :params:
 549             - base_directory: is the location of the package to be created
 550             - name: is the name of the package
 551             - override: is the flag used to indicate the creation of the list even if the file exist to override it
 552         :return: Files and Folders not found. In case of override, it will return all file list
 553         """
 554         self._logger.debug("")
 555         prefix = "{}/{}".format(base_directory, name)
 556         files_folders = {
 557             "folders": [
 558                 ("{}_ns".format(prefix), "ns"),
 559                 ("{}_ns/icons".format(prefix), "ns"),
 560                 ("{}_ns/charms".format(prefix), "ns"),
 561                 ("{}_vnf".format(name), "vnf"),
 562                 ("{}_vnf/charms".format(prefix), "vnf"),
 563                 ("{}_vnf/cloud_init".format(prefix), "vnf"),
 564                 ("{}_vnf/images".format(prefix), "vnf"),
 565                 ("{}_vnf/icons".format(prefix), "vnf"),
 566                 ("{}_vnf/scripts".format(prefix), "vnf"),
 567                 ("{}_nst".format(prefix), "nst"),
 568                 ("{}_nst/icons".format(prefix), "nst"),
 569             ],
 570             "files": [
 571                 ("{}_ns/{}_nsd.yaml".format(prefix, name), "ns", "descriptor"),
 572                 ("{}_ns/README.md".format(prefix), "ns", "readme"),
 573                 ("{}_vnf/{}_vnfd.yaml".format(prefix, name), "vnf", "descriptor"),
 574                 (
 575                     "{}_vnf/cloud_init/cloud-config.txt".format(prefix),
 576                     "vnf",
 577                     "cloud_init",
 578                 ),
 579                 ("{}_vnf/README.md".format(prefix), "vnf", "readme"),
 580                 ("{}_nst/{}_nst.yaml".format(prefix, name), "nst", "descriptor"),
 581                 ("{}_nst/README.md".format(prefix), "nst", "readme"),
 582             ],
 583         }
 584         missing_files_folders = self.check_files_folders(files_folders, override)
 585         # print("Missing files and folders: {}".format(missing_files_folders))
 586         return missing_files_folders
 587
 588     def charm_build(self, charms_folder, build_name):
 589         """
 590         Build the charms inside the package.
 591         params: package_folder is the name of the folder where is the charms to compile.
 592                 build_name is the name of the layer or interface
 593         """
 594         self._logger.debug("")
 595         os.environ["JUJU_REPOSITORY"] = "{}/charms".format(charms_folder)
 596         os.environ["CHARM_LAYERS_DIR"] = "{}/layers".format(
 597             os.environ["JUJU_REPOSITORY"]
 598         )
 599         os.environ["CHARM_INTERFACES_DIR"] = "{}/interfaces".format(
 600             os.environ["JUJU_REPOSITORY"]
 601         )
 602         os.environ["CHARM_BUILD_DIR"] = "{}/charms/builds".format(charms_folder)
 603         if not os.path.exists(os.environ["CHARM_BUILD_DIR"]):
 604             os.makedirs(os.environ["CHARM_BUILD_DIR"])
 605         src_folder = "{}/{}".format(os.environ["CHARM_LAYERS_DIR"], build_name)
 606         result = subprocess.run(["charm", "build", "{}".format(src_folder)])
 607         if result.returncode == 1:
 608             raise ClientException("failed to build the charm: {}".format(src_folder))
 609         self._logger.verbose("charm {} built".format(src_folder))
 610
 611     def charmcraft_build(self, package_folder, charm_name):
 612         """
 613         Build the charms inside the package (new operator framework charms)
 614         params: package_folder is the name of the folder where is the charms to compile.
 615                 build_name is the name of the layer or interface
 616         """
 617         self._logger.debug("Building charm {}".format(charm_name))
 618         src_folder = f"{package_folder}/charms/ops/{charm_name}"
 619         current_directory = os.getcwd()
 620         os.chdir(src_folder)
 621         try:
 622             result = subprocess.run(["charmcraft", "build"])
 623             if result.returncode == 1:
 624                 raise ClientException(
 625                     "failed to build the charm: {}".format(src_folder)
 626                 )
 627             subprocess.run(["rm", "-rf", f"../../{charm_name}"])
 628             subprocess.run(["mv", "build", f"../../{charm_name}"])
 629             self._logger.verbose("charm {} built".format(src_folder))
 630         finally:
 631             os.chdir(current_directory)
 632
 633     def build_tarfile(self, package_folder, charm_list=None):
 634         """
 635         Creates a .tar.gz file given a package_folder
 636         params: package_folder is the name of the folder to be packaged
 637         returns: .tar.gz name
 638         """
 639         self._logger.debug("")
 640         cwd = None
 641         try:
 642             directory_name, package_name = self.create_temp_dir(
 643                 package_folder, charm_list
 644             )
 645             cwd = os.getcwd()
 646             os.chdir(directory_name)
 647             self.calculate_checksum(package_name)
 648             with tarfile.open("{}.tar.gz".format(package_name), mode="w:gz") as archive:
 649                 print("Adding File: {}".format(package_name))
 650                 archive.add("{}".format(package_name), recursive=True)
 651             # return "Created {}.tar.gz".format(package_folder)
 652             # self.build("{}".format(os.path.basename(package_folder)))
 653             os.chdir(cwd)
 654             cwd = None
 655             created_package = "{}/{}.tar.gz".format(
 656                 os.path.dirname(package_folder) or ".", package_name
 657             )
 658             os.rename(
 659                 "{}/{}.tar.gz".format(directory_name, package_name), created_package
 660             )
 661             os.rename(
 662                 "{}/{}/checksums.txt".format(directory_name, package_name),
 663                 "{}/checksums.txt".format(package_folder),
 664             )
 665             print("Package created: {}".format(created_package))
 666             return created_package
 667         except Exception as exc:
 668             raise ClientException(
 669                 "failure during build of targz file (create temp dir, calculate checksum, "
 670                 "tar.gz file): {}".format(exc)
 671             )
 672         finally:
 673             if cwd:
 674                 os.chdir(cwd)
 675             shutil.rmtree(os.path.join(package_folder, "tmp"))
 676
 677     def create_temp_dir(self, package_folder, charm_list=None):
 678         """
 679         Method to create a temporary folder where we can move the files in package_folder
 680         """
 681         self._logger.debug("")
 682         ignore_patterns = ".gitignore"
 683         ignore = shutil.ignore_patterns(ignore_patterns)
 684         directory_name = os.path.abspath(package_folder)
 685         package_name = os.path.basename(directory_name)
 686         directory_name += "/tmp"
 687         os.makedirs("{}/{}".format(directory_name, package_name), exist_ok=True)
 688         self._logger.debug("Makedirs DONE: {}/{}".format(directory_name, package_name))
 689         for item in os.listdir(package_folder):
 690             self._logger.debug("Item: {}".format(item))
 691             if item != "tmp":
 692                 s = os.path.join(package_folder, item)
 693                 d = os.path.join(os.path.join(directory_name, package_name), item)
 694                 if os.path.isdir(s):
 695                     if item == "charms":
 696                         os.makedirs(d, exist_ok=True)
 697                         s_builds = os.path.join(s, "builds")
 698                         for charm in charm_list:
 699                             self._logger.debug("Copying charm {}".format(charm))
 700                             if charm in os.listdir(s):
 701                                 s_charm = os.path.join(s, charm)
 702                             elif charm in os.listdir(s_builds):
 703                                 s_charm = os.path.join(s_builds, charm)
 704                             else:
 705                                 raise ClientException(
 706                                     "The charm {} referenced in the descriptor file "
 707                                     "could not be found in {}/charms or in {}/charms/builds".format(
 708                                         charm, package_folder, package_folder
 709                                     )
 710                                 )
 711                             d_temp = os.path.join(d, charm)
 712                             self._logger.debug(
 713                                 "Copying tree: {} -> {}".format(s_charm, d_temp)
 714                             )
 715                             shutil.copytree(
 716                                 s_charm, d_temp, symlinks=True, ignore=ignore
 717                             )
 718                             self._logger.debug("DONE")
 719                     else:
 720                         self._logger.debug("Copying tree: {} -> {}".format(s, d))
 721                         shutil.copytree(s, d, symlinks=True, ignore=ignore)
 722                         self._logger.debug("DONE")
 723                 else:
 724                     if item in ignore_patterns:
 725                         continue
 726                     self._logger.debug("Copying file: {} -> {}".format(s, d))
 727                     shutil.copy2(s, d)
 728                     self._logger.debug("DONE")
 729         return directory_name, package_name
 730
 731     def charms_search(self, descriptor_file, desc_type):
 732         self._logger.debug(
 733             "descriptor_file: {}, desc_type: {}".format(descriptor_file, desc_type)
 734         )
 735         charms_set = set()
 736         with open("{}".format(descriptor_file)) as yaml_desc:
 737             descriptor_dict = yaml.safe_load(yaml_desc)
 738             # self._logger.debug("\n"+yaml.safe_dump(descriptor_dict, indent=4, default_flow_style=False))
 739
 740             if (
 741                 desc_type == "vnf"
 742                 and (
 743                     "vnfd:vnfd-catalog" in descriptor_dict
 744                     or "vnfd-catalog" in descriptor_dict
 745                 )
 746             ) or (
 747                 desc_type == "ns"
 748                 and (
 749                     "nsd:nsd-catalog" in descriptor_dict
 750                     or "nsd-catalog" in descriptor_dict
 751                 )
 752             ):
 753                 charms_set = self._charms_search_on_osm_im_dict(
 754                     descriptor_dict, desc_type
 755                 )
 756             else:
 757                 if desc_type == "ns":
 758                     get_charm_list = self._charms_search_on_nsd_sol006_dict
 759                 elif desc_type == "vnf":
 760                     get_charm_list = self._charms_search_on_vnfd_sol006_dict
 761                 else:
 762                     raise Exception("Bad descriptor type")
 763                 charms_set = get_charm_list(descriptor_dict)
 764         return charms_set
 765
 766     def _charms_search_on_osm_im_dict(self, osm_im_dict, desc_type):
 767         self._logger.debug("")
 768         charms_set = []
 769         for k1, v1 in osm_im_dict.items():
 770             for k2, v2 in v1.items():
 771                 for entry in v2:
 772                     if "{}-configuration".format(desc_type) in entry:
 773                         vnf_config = entry["{}-configuration".format(desc_type)]
 774                         for k3, v3 in vnf_config.items():
 775                             if "charm" in v3:
 776                                 charms_set.add((v3["charm"]))
 777                     if "vdu" in entry:
 778                         vdus = entry["vdu"]
 779                         for vdu in vdus:
 780                             if "vdu-configuration" in vdu:
 781                                 for k4, v4 in vdu["vdu-configuration"].items():
 782                                     if "charm" in v4:
 783                                         charms_set.add((v4["charm"]))
 784         return charms_set
 785
 786     def _charms_search_on_vnfd_sol006_dict(self, sol006_dict):
 787         self._logger.debug("")
 788         charms_set = set()
 789         dfs = sol006_dict.get("vnfd", {}).get("df", [])
 790         for df in dfs:
 791             day_1_2s = (
 792                 df.get("lcm-operations-configuration", {})
 793                 .get("operate-vnf-op-config", {})
 794                 .get("day1-2")
 795             )
 796             if day_1_2s is not None:
 797                 for day_1_2 in day_1_2s:
 798                     exec_env_list = day_1_2.get("execution-environment-list", [])
 799                     for exec_env in exec_env_list:
 800                         if "juju" in exec_env and "charm" in exec_env["juju"]:
 801                             charms_set.add(exec_env["juju"]["charm"])
 802         return charms_set
 803
 804     def _charms_search_on_nsd_sol006_dict(self, sol006_dict):
 805         self._logger.debug("")
 806         charms_set = set()
 807         nsd_list = sol006_dict.get("nsd", {}).get("nsd", [])
 808         for nsd in nsd_list:
 809             charm = nsd.get("ns-configuration", {}).get("juju", {}).get("charm")
 810             if charm:
 811                 charms_set.add(charm)
 812         return charms_set