osmclient/common/package_tool.py

   1 # /bin/env python3
   2 # Copyright 2019 ATOS
   3 #
   4 # All Rights Reserved.
   5 #
   6 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 #    not use this file except in compliance with the License. You may obtain
   8 #    a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #    Unless required by applicable law or agreed to in writing, software
  13 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 #    License for the specific language governing permissions and limitations
  16 #    under the License.
  17
  18 import glob
  19 import hashlib
  20 import logging
  21 import os
  22 import shutil
  23 import subprocess
  24 import tarfile
  25 import time
  26
  27 from jinja2 import Environment, PackageLoader
  28 from osm_im.validation import Validation as validation_im
  29 from osm_im.validation import ValidationException
  30 from osm_im import im_translation
  31 from osmclient.common.exceptions import ClientException
  32 import yaml
  33
  34
  35 class PackageTool(object):
  36     def __init__(self, client=None):
  37         self._client = client
  38         self._logger = logging.getLogger("osmclient")
  39         self._validator = validation_im()
  40
  41     def create(
  42         self,
  43         package_type,
  44         base_directory,
  45         package_name,
  46         override,
  47         image,
  48         vdus,
  49         vcpu,
  50         memory,
  51         storage,
  52         interfaces,
  53         vendor,
  54         detailed,
  55         netslice_subnets,
  56         netslice_vlds,
  57         old,
  58     ):
  59         """
  60         **Create a package descriptor**
  61
  62         :params:
  63             - package_type: [vnf, ns, nst]
  64             - base directory: path of destination folder
  65             - package_name: is the name of the package to be created
  66             - image: specify the image of the vdu
  67             - vcpu: number of virtual cpus of the vdu
  68             - memory: amount of memory in MB pf the vdu
  69             - storage: amount of storage in GB of the vdu
  70             - interfaces: number of interfaces besides management interface
  71             - vendor: vendor name of the vnf/ns
  72             - detailed: include all possible values for NSD, VNFD, NST
  73             - netslice_subnets: number of netslice_subnets for the NST
  74             - netslice_vlds: number of virtual link descriptors for the NST
  75             - old: flag to create a descriptor using the previous OSM format (pre SOL006, OSM<9)
  76
  77         :return: status
  78         """
  79         self._logger.debug("")
  80         # print("location: {}".format(osmclient.__path__))
  81         file_loader = PackageLoader("osmclient")
  82         env = Environment(loader=file_loader)
  83         if package_type == "ns":
  84             template = env.get_template("nsd.yaml.j2" if not old else "nsd_old.yaml.j2")
  85             content = {
  86                 "name": package_name,
  87                 "vendor": vendor,
  88                 "vdus": vdus,
  89                 "clean": False,
  90                 "interfaces": interfaces,
  91                 "detailed": detailed,
  92             }
  93         elif package_type == "vnf":
  94             template = env.get_template(
  95                 "vnfd.yaml.j2" if not old else "vnfd_old.yaml.j2"
  96             )
  97             content = {
  98                 "name": package_name,
  99                 "vendor": vendor,
 100                 "vdus": vdus,
 101                 "clean": False,
 102                 "interfaces": interfaces,
 103                 "image": image,
 104                 "vcpu": vcpu,
 105                 "memory": memory,
 106                 "storage": storage,
 107                 "detailed": detailed,
 108             }
 109         elif package_type == "nst":
 110             # TODO: repo-index did not support nst in OSM<9, no changes in template
 111             template = env.get_template("nst.yaml.j2")
 112             content = {
 113                 "name": package_name,
 114                 "vendor": vendor,
 115                 "interfaces": interfaces,
 116                 "netslice_subnets": netslice_subnets,
 117                 "netslice_vlds": netslice_vlds,
 118                 "detailed": detailed,
 119             }
 120         else:
 121             raise ClientException(
 122                 "Wrong descriptor type {}. Options: ns, vnf, nst".format(package_type)
 123             )
 124
 125         self._logger.debug("To be rendered: {}".format(content))
 126         output = template.render(content)
 127         self._logger.debug(output)
 128
 129         structure = self.discover_folder_structure(
 130             base_directory, package_name, override
 131         )
 132         if structure.get("folders"):
 133             self.create_folders(structure["folders"], package_type)
 134         if structure.get("files"):
 135             self.create_files(structure["files"], output, package_type)
 136         return "Created"
 137
 138     def validate(self, base_directory, recursive=True, old_format=False):
 139         """
 140         **Validate OSM Descriptors given a path**
 141
 142         :params:
 143             - base_directory is the root path for all descriptors
 144
 145         :return: List of dict of validated descriptors. keys: type, path, valid, error
 146         """
 147         self._logger.debug("")
 148         table = []
 149         if recursive:
 150             descriptors_paths = [
 151                 f for f in glob.glob(base_directory + "/**/*.yaml", recursive=recursive)
 152             ]
 153         else:
 154             descriptors_paths = [
 155                 f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)
 156             ]
 157         self._logger.info("Base directory: {}".format(base_directory))
 158         self._logger.info(
 159             "{} Descriptors found to validate".format(len(descriptors_paths))
 160         )
 161         for desc_path in descriptors_paths:
 162             with open(desc_path) as descriptor_file:
 163                 descriptor_data = descriptor_file.read()
 164             desc_type = "-"
 165             try:
 166                 # TODO: refactor validation_im.yaml_validation to @staticmethod
 167                 desc_type, descriptor_data = validation_im.yaml_validation(
 168                     self, descriptor_data
 169                 )
 170                 self._logger.debug(f"Validate {desc_type} {descriptor_data}")
 171                 if not old_format:
 172                     if desc_type == "vnfd" or desc_type == "nsd":
 173                         self._logger.error(
 174                             "OSM descriptor '{}' written in an unsupported format. Please update to ETSI SOL006 format".format(
 175                                 desc_path
 176                             )
 177                         )
 178                         self._logger.warning(
 179                             "Package validation skipped. It can still be done with 'osm package-validate --old'"
 180                         )
 181                         self._logger.warning(
 182                             "Package build can still be done with 'osm package-build --skip-validation'"
 183                         )
 184                         raise Exception("Not SOL006 format")
 185                 validation_im.pyangbind_validation(self, desc_type, descriptor_data)
 186                 table.append(
 187                     {"type": desc_type, "path": desc_path, "valid": "OK", "error": "-"}
 188                 )
 189             except Exception as e:
 190                 self._logger.error(f"Validation error: {e}", exc_info=True)
 191                 table.append(
 192                     {
 193                         "type": desc_type,
 194                         "path": desc_path,
 195                         "valid": "ERROR",
 196                         "error": str(e),
 197                     }
 198                 )
 199             self._logger.debug(table[-1])
 200         return table
 201
 202     def translate(self, base_directory, recursive=True, dryrun=False):
 203         """
 204         **Translate OSM Packages given a path**
 205
 206         :params:
 207             - base_directory is the root path for all packages
 208
 209         :return: List of dict of translated packages. keys: current type, new type, path, valid, translated, error
 210         """
 211         self._logger.debug("")
 212         table = []
 213         if recursive:
 214             descriptors_paths = [
 215                 f for f in glob.glob(base_directory + "/**/*.yaml", recursive=recursive)
 216             ]
 217         else:
 218             descriptors_paths = [
 219                 f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)
 220             ]
 221         print("Base directory: {}".format(base_directory))
 222         print("{} Descriptors found to validate".format(len(descriptors_paths)))
 223         for desc_path in descriptors_paths:
 224             with open(desc_path) as descriptor_file:
 225                 descriptor_data = descriptor_file.read()
 226             desc_type = "-"
 227             try:
 228                 desc_type, descriptor_data = validation_im.yaml_validation(
 229                     self, descriptor_data
 230                 )
 231                 self._logger.debug("desc_type: {}".format(desc_type))
 232                 self._logger.debug("descriptor_data:\n{}".format(descriptor_data))
 233                 self._validator.pyangbind_validation(desc_type, descriptor_data)
 234                 if not (desc_type == "vnfd" or desc_type == "nsd"):
 235                     table.append(
 236                         {
 237                             "current type": desc_type,
 238                             "new type": desc_type,
 239                             "path": desc_path,
 240                             "valid": "OK",
 241                             "translated": "N/A",
 242                             "error": "-",
 243                         }
 244                     )
 245                 else:
 246                     new_desc_type = desc_type
 247                     try:
 248                         sol006_model = yaml.safe_dump(
 249                             im_translation.translate_im_model_to_sol006(
 250                                 descriptor_data
 251                             ),
 252                             indent=4,
 253                             default_flow_style=False,
 254                         )
 255                         (
 256                             new_desc_type,
 257                             new_descriptor_data,
 258                         ) = self._validator.yaml_validation(sol006_model)
 259                         self._validator.pyangbind_validation(
 260                             new_desc_type, new_descriptor_data
 261                         )
 262                         if not dryrun:
 263                             with open(desc_path, "w") as descriptor_file:
 264                                 descriptor_file.write(sol006_model)
 265                         table.append(
 266                             {
 267                                 "current type": desc_type,
 268                                 "new type": new_desc_type,
 269                                 "path": desc_path,
 270                                 "valid": "OK",
 271                                 "translated": "OK",
 272                                 "error": "-",
 273                             }
 274                         )
 275                     except ValidationException as ve2:
 276                         table.append(
 277                             {
 278                                 "current type": desc_type,
 279                                 "new type": new_desc_type,
 280                                 "path": desc_path,
 281                                 "valid": "OK",
 282                                 "translated": "ERROR",
 283                                 "error": "Error in the post-validation: {}".format(
 284                                     str(ve2)
 285                                 ),
 286                             }
 287                         )
 288                     except Exception as e2:
 289                         table.append(
 290                             {
 291                                 "current type": desc_type,
 292                                 "new type": new_desc_type,
 293                                 "path": desc_path,
 294                                 "valid": "OK",
 295                                 "translated": "ERROR",
 296                                 "error": "Error in the translation: {}".format(str(e2)),
 297                             }
 298                         )
 299             except ValidationException as ve:
 300                 table.append(
 301                     {
 302                         "current type": desc_type,
 303                         "new type": "N/A",
 304                         "path": desc_path,
 305                         "valid": "ERROR",
 306                         "translated": "N/A",
 307                         "error": "Error in the pre-validation: {}".format(str(ve)),
 308                     }
 309                 )
 310             except Exception as e:
 311                 table.append(
 312                     {
 313                         "current type": desc_type,
 314                         "new type": "N/A",
 315                         "path": desc_path,
 316                         "valid": "ERROR",
 317                         "translated": "N/A",
 318                         "error": str(e),
 319                     }
 320                 )
 321         return table
 322
 323     def descriptor_translate(self, descriptor_file):
 324         """
 325         **Translate input descriptor file from Rel EIGHT OSM to SOL006**
 326
 327         :params:
 328             - base_directory is the root path for all packages
 329
 330         :return: YAML descriptor in the new format
 331         """
 332         self._logger.debug("")
 333         with open(descriptor_file, "r") as df:
 334             im_model = yaml.safe_load(df.read())
 335         sol006_model = im_translation.translate_im_model_to_sol006(im_model)
 336         return yaml.safe_dump(sol006_model, indent=4, default_flow_style=False)
 337
 338     def build(self, package_folder, skip_validation=False, skip_charm_build=False):
 339         """
 340         **Creates a .tar.gz file given a package_folder**
 341
 342         :params:
 343             - package_folder: is the name of the folder to be packaged
 344             - skip_validation: is the flag to validate or not the descriptors on the folder before build
 345
 346         :returns: message result for the build process
 347         """
 348         self._logger.debug("")
 349         package_folder = package_folder.rstrip("/")
 350         if not os.path.exists("{}".format(package_folder)):
 351             return "Fail, package is not in the specified path"
 352         if not skip_validation:
 353             print("Validating package {}".format(package_folder))
 354             results = self.validate(package_folder, recursive=False)
 355             if results:
 356                 for result in results:
 357                     if result["valid"] != "OK":
 358                         raise ClientException(
 359                             "There was an error validating the file {} with error: {}".format(
 360                                 result["path"], result["error"]
 361                             )
 362                         )
 363                 print("Validation OK")
 364             else:
 365                 raise ClientException(
 366                     "No descriptor file found in: {}".format(package_folder)
 367                 )
 368         charm_list = self.build_all_charms(package_folder, skip_charm_build)
 369         return self.build_tarfile(package_folder, charm_list)
 370
 371     def calculate_checksum(self, package_folder):
 372         """
 373         **Function to calculate the checksum given a folder**
 374
 375         :params:
 376             - package_folder: is the folder where we have the files to calculate the checksum
 377         :returns: None
 378         """
 379         self._logger.debug("")
 380         files = [
 381             f
 382             for f in glob.glob(package_folder + "/**/*.*", recursive=True)
 383             if os.path.isfile(f)
 384         ]
 385         with open("{}/checksums.txt".format(package_folder), "w+") as checksum:
 386             for file_item in files:
 387                 if "checksums.txt" in file_item:
 388                     continue
 389                 # from https://www.quickprogrammingtips.com/python/how-to-calculate-md5-hash-of-a-file-in-python.html
 390                 md5_hash = hashlib.md5()
 391                 with open(file_item, "rb") as f:
 392                     # Read and update hash in chunks of 4K
 393                     for byte_block in iter(lambda: f.read(4096), b""):
 394                         md5_hash.update(byte_block)
 395                     checksum.write("{}\t{}\n".format(md5_hash.hexdigest(), file_item))
 396
 397     def create_folders(self, folders, package_type):
 398         """
 399         **Create folder given a list of folders**
 400
 401         :params:
 402             - folders: [List] list of folders paths to be created
 403             - package_type: is the type of package to be created
 404         :return: None
 405         """
 406         self._logger.debug("")
 407         for folder in folders:
 408             try:
 409                 # print("Folder {} == package_type {}".format(folder[1], package_type))
 410                 if folder[1] == package_type:
 411                     print("Creating folder:\t{}".format(folder[0]))
 412                     os.makedirs(folder[0])
 413             except FileExistsError:
 414                 pass
 415
 416     def save_file(self, file_name, file_body):
 417         """
 418         **Create a file given a name and the content**
 419
 420         :params:
 421             - file_name: is the name of the file with the relative route
 422             - file_body: is the content of the file
 423         :return: None
 424         """
 425         self._logger.debug("")
 426         print("Creating file:  \t{}".format(file_name))
 427         try:
 428             with open(file_name, "w+") as f:
 429                 f.write(file_body)
 430         except Exception as e:
 431             raise ClientException(e)
 432
 433     def generate_readme(self):
 434         """
 435         **Creates the README content**
 436
 437         :returns: readme content
 438         """
 439         self._logger.debug("")
 440         return """# Descriptor created by OSM descriptor package generated\n\n**Created on {} **""".format(
 441             time.strftime("%m/%d/%Y, %H:%M:%S", time.localtime())
 442         )
 443
 444     def generate_cloud_init(self):
 445         """
 446         **Creates the cloud-init content**
 447
 448         :returns: cloud-init content
 449         """
 450         self._logger.debug("")
 451         return "---\n#cloud-config"
 452
 453     def create_files(self, files, file_content, package_type):
 454         """
 455         **Creates the files given the file list and type**
 456
 457         :params:
 458             - files: is the list of files structure
 459             - file_content: is the content of the descriptor rendered by the template
 460             - package_type: is the type of package to filter the creation structure
 461
 462         :return: None
 463         """
 464         self._logger.debug("")
 465         for file_item, file_package, file_type in files:
 466             if package_type == file_package:
 467                 if file_type == "descriptor":
 468                     self.save_file(file_item, file_content)
 469                 elif file_type == "readme":
 470                     self.save_file(file_item, self.generate_readme())
 471                 elif file_type == "cloud_init":
 472                     self.save_file(file_item, self.generate_cloud_init())
 473
 474     def check_files_folders(self, path_list, override):
 475         """
 476         **Find files and folders missing given a directory structure {"folders": [], "files": []}**
 477
 478         :params:
 479             - path_list: is the list of files and folders to be created
 480             - override: is the flag used to indicate the creation of the list even if the file exist to override it
 481
 482         :return: Missing paths Dict
 483         """
 484         self._logger.debug("")
 485         missing_paths = {}
 486         folders = []
 487         files = []
 488         for folder in path_list.get("folders"):
 489             if not os.path.exists(folder[0]):
 490                 folders.append(folder)
 491         missing_paths["folders"] = folders
 492
 493         for file_item in path_list.get("files"):
 494             if not os.path.exists(file_item[0]) or override is True:
 495                 files.append(file_item)
 496         missing_paths["files"] = files
 497
 498         return missing_paths
 499
 500     def build_all_charms(self, package_folder, skip_charm_build):
 501         """
 502         **Read the descriptor file, check that the charms referenced are in the folder and compiles them**
 503
 504         :params:
 505             - packet_folder: is the location of the package
 506         :return: Files and Folders not found. In case of override, it will return all file list
 507         """
 508         self._logger.debug("")
 509         charms_set = set()
 510         descriptor_file = False
 511         descriptors_paths = [f for f in glob.glob(package_folder + "/*.yaml")]
 512         for file in descriptors_paths:
 513             if file.endswith("nfd.yaml"):
 514                 descriptor_file = True
 515                 charms_set = self.charms_search(file, "vnf")
 516             if file.endswith("nsd.yaml"):
 517                 descriptor_file = True
 518                 charms_set = self.charms_search(file, "ns")
 519         print("List of charms in the descriptor: {}".format(charms_set))
 520         if not descriptor_file:
 521             raise ClientException(
 522                 'Descriptor filename is not correct in: {}. It should end with "nfd.yaml" or "nsd.yaml"'.format(
 523                     package_folder
 524                 )
 525             )
 526         if charms_set and not skip_charm_build:
 527             for charmName in charms_set:
 528                 if os.path.isdir(
 529                     "{}/charms/layers/{}".format(package_folder, charmName)
 530                 ):
 531                     print(
 532                         "Building charm {}/charms/layers/{}".format(
 533                             package_folder, charmName
 534                         )
 535                     )
 536                     self.charm_build(package_folder, charmName)
 537                     print("Charm built: {}".format(charmName))
 538                 elif os.path.isdir(
 539                     "{}/charms/ops/{}".format(package_folder, charmName)
 540                 ):
 541                     self.charmcraft_build(package_folder, charmName)
 542                 else:
 543                     if not os.path.isdir(
 544                         "{}/charms/{}".format(package_folder, charmName)
 545                     ) and not os.path.isfile(
 546                         "{}/charms/{}".format(package_folder, charmName)
 547                     ):
 548                         raise ClientException(
 549                             "The charm: {} referenced in the descriptor file "
 550                             "is not present either in {}/charms or in {}/charms/layers".format(
 551                                 charmName, package_folder, package_folder
 552                             )
 553                         )
 554         self._logger.debug("Return list of charms: {}".format(charms_set))
 555         return charms_set
 556
 557     def discover_folder_structure(self, base_directory, name, override):
 558         """
 559         **Discover files and folders structure for OSM descriptors given a base_directory and name**
 560
 561         :params:
 562             - base_directory: is the location of the package to be created
 563             - name: is the name of the package
 564             - override: is the flag used to indicate the creation of the list even if the file exist to override it
 565         :return: Files and Folders not found. In case of override, it will return all file list
 566         """
 567         self._logger.debug("")
 568         prefix = "{}/{}".format(base_directory, name)
 569         files_folders = {
 570             "folders": [
 571                 ("{}_ns".format(prefix), "ns"),
 572                 ("{}_ns/icons".format(prefix), "ns"),
 573                 ("{}_ns/charms".format(prefix), "ns"),
 574                 ("{}_vnf".format(name), "vnf"),
 575                 ("{}_vnf/charms".format(prefix), "vnf"),
 576                 ("{}_vnf/cloud_init".format(prefix), "vnf"),
 577                 ("{}_vnf/images".format(prefix), "vnf"),
 578                 ("{}_vnf/icons".format(prefix), "vnf"),
 579                 ("{}_vnf/scripts".format(prefix), "vnf"),
 580                 ("{}_nst".format(prefix), "nst"),
 581                 ("{}_nst/icons".format(prefix), "nst"),
 582             ],
 583             "files": [
 584                 ("{}_ns/{}_nsd.yaml".format(prefix, name), "ns", "descriptor"),
 585                 ("{}_ns/README.md".format(prefix), "ns", "readme"),
 586                 ("{}_vnf/{}_vnfd.yaml".format(prefix, name), "vnf", "descriptor"),
 587                 (
 588                     "{}_vnf/cloud_init/cloud-config.txt".format(prefix),
 589                     "vnf",
 590                     "cloud_init",
 591                 ),
 592                 ("{}_vnf/README.md".format(prefix), "vnf", "readme"),
 593                 ("{}_nst/{}_nst.yaml".format(prefix, name), "nst", "descriptor"),
 594                 ("{}_nst/README.md".format(prefix), "nst", "readme"),
 595             ],
 596         }
 597         missing_files_folders = self.check_files_folders(files_folders, override)
 598         # print("Missing files and folders: {}".format(missing_files_folders))
 599         return missing_files_folders
 600
 601     def charm_build(self, charms_folder, build_name):
 602         """
 603         Build the charms inside the package.
 604         params: package_folder is the name of the folder where is the charms to compile.
 605                 build_name is the name of the layer or interface
 606         """
 607         self._logger.debug("")
 608         os.environ["JUJU_REPOSITORY"] = "{}/charms".format(charms_folder)
 609         os.environ["CHARM_LAYERS_DIR"] = "{}/layers".format(
 610             os.environ["JUJU_REPOSITORY"]
 611         )
 612         os.environ["CHARM_INTERFACES_DIR"] = "{}/interfaces".format(
 613             os.environ["JUJU_REPOSITORY"]
 614         )
 615         os.environ["CHARM_BUILD_DIR"] = "{}/charms/builds".format(charms_folder)
 616         if not os.path.exists(os.environ["CHARM_BUILD_DIR"]):
 617             os.makedirs(os.environ["CHARM_BUILD_DIR"])
 618         src_folder = "{}/{}".format(os.environ["CHARM_LAYERS_DIR"], build_name)
 619         result = subprocess.run(["charm", "build", "{}".format(src_folder)])
 620         if result.returncode == 1:
 621             raise ClientException("failed to build the charm: {}".format(src_folder))
 622         self._logger.verbose("charm {} built".format(src_folder))
 623
 624     def charmcraft_build(self, package_folder, charm_name):
 625         """
 626         Build the charms inside the package (new operator framework charms)
 627         params: package_folder is the name of the folder where is the charms to compile.
 628                 build_name is the name of the layer or interface
 629         """
 630         self._logger.debug("Building charm {}".format(charm_name))
 631         src_folder = f"{package_folder}/charms/ops/{charm_name}"
 632         current_directory = os.getcwd()
 633         os.chdir(src_folder)
 634         try:
 635             result = subprocess.run(["charmcraft", "build"])
 636             if result.returncode == 1:
 637                 raise ClientException(
 638                     "failed to build the charm: {}".format(src_folder)
 639                 )
 640             subprocess.run(["rm", "-rf", f"../../{charm_name}"])
 641             subprocess.run(["mv", "build", f"../../{charm_name}"])
 642             self._logger.verbose("charm {} built".format(src_folder))
 643         finally:
 644             os.chdir(current_directory)
 645
 646     def build_tarfile(self, package_folder, charm_list=None):
 647         """
 648         Creates a .tar.gz file given a package_folder
 649         params: package_folder is the name of the folder to be packaged
 650         returns: .tar.gz name
 651         """
 652         self._logger.debug("")
 653         cwd = None
 654         try:
 655             directory_name, package_name = self.create_temp_dir(
 656                 package_folder, charm_list
 657             )
 658             cwd = os.getcwd()
 659             os.chdir(directory_name)
 660             self.calculate_checksum(package_name)
 661             with tarfile.open("{}.tar.gz".format(package_name), mode="w:gz") as archive:
 662                 print("Adding File: {}".format(package_name))
 663                 archive.add("{}".format(package_name), recursive=True)
 664             # return "Created {}.tar.gz".format(package_folder)
 665             # self.build("{}".format(os.path.basename(package_folder)))
 666             os.chdir(cwd)
 667             cwd = None
 668             created_package = "{}/{}.tar.gz".format(
 669                 os.path.dirname(package_folder) or ".", package_name
 670             )
 671             os.rename(
 672                 "{}/{}.tar.gz".format(directory_name, package_name), created_package
 673             )
 674             os.rename(
 675                 "{}/{}/checksums.txt".format(directory_name, package_name),
 676                 "{}/checksums.txt".format(package_folder),
 677             )
 678             print("Package created: {}".format(created_package))
 679             return created_package
 680         except Exception as exc:
 681             raise ClientException(
 682                 "failure during build of targz file (create temp dir, calculate checksum, "
 683                 "tar.gz file): {}".format(exc)
 684             )
 685         finally:
 686             if cwd:
 687                 os.chdir(cwd)
 688             shutil.rmtree(os.path.join(package_folder, "tmp"))
 689
 690     def create_temp_dir(self, package_folder, charm_list=None):
 691         """
 692         Method to create a temporary folder where we can move the files in package_folder
 693         """
 694         self._logger.debug("")
 695         ignore_patterns = ".gitignore"
 696         ignore = shutil.ignore_patterns(ignore_patterns)
 697         directory_name = os.path.abspath(package_folder)
 698         package_name = os.path.basename(directory_name)
 699         directory_name += "/tmp"
 700         os.makedirs("{}/{}".format(directory_name, package_name), exist_ok=True)
 701         self._logger.debug("Makedirs DONE: {}/{}".format(directory_name, package_name))
 702         for item in os.listdir(package_folder):
 703             self._logger.debug("Item: {}".format(item))
 704             if item != "tmp":
 705                 s = os.path.join(package_folder, item)
 706                 d = os.path.join(os.path.join(directory_name, package_name), item)
 707                 if os.path.isdir(s):
 708                     if item == "charms":
 709                         os.makedirs(d, exist_ok=True)
 710                         s_builds = os.path.join(s, "builds")
 711                         for charm in charm_list:
 712                             self._logger.debug("Copying charm {}".format(charm))
 713                             if charm in os.listdir(s):
 714                                 s_charm = os.path.join(s, charm)
 715                             elif charm in os.listdir(s_builds):
 716                                 s_charm = os.path.join(s_builds, charm)
 717                             else:
 718                                 raise ClientException(
 719                                     "The charm {} referenced in the descriptor file "
 720                                     "could not be found in {}/charms or in {}/charms/builds".format(
 721                                         charm, package_folder, package_folder
 722                                     )
 723                                 )
 724                             d_temp = os.path.join(d, charm)
 725                             self._logger.debug(
 726                                 "Copying tree: {} -> {}".format(s_charm, d_temp)
 727                             )
 728                             if os.path.isdir(s_charm):
 729                                 shutil.copytree(
 730                                     s_charm, d_temp, symlinks=True, ignore=ignore
 731                                 )
 732                             else:
 733                                 shutil.copy2(s_charm, d_temp)
 734                             self._logger.debug("DONE")
 735                     else:
 736                         self._logger.debug("Copying tree: {} -> {}".format(s, d))
 737                         shutil.copytree(s, d, symlinks=True, ignore=ignore)
 738                         self._logger.debug("DONE")
 739                 else:
 740                     if item in ignore_patterns:
 741                         continue
 742                     self._logger.debug("Copying file: {} -> {}".format(s, d))
 743                     shutil.copy2(s, d)
 744                     self._logger.debug("DONE")
 745         return directory_name, package_name
 746
 747     def charms_search(self, descriptor_file, desc_type):
 748         self._logger.debug(
 749             "descriptor_file: {}, desc_type: {}".format(descriptor_file, desc_type)
 750         )
 751         charms_set = set()
 752         with open("{}".format(descriptor_file)) as yaml_desc:
 753             descriptor_dict = yaml.safe_load(yaml_desc)
 754             # self._logger.debug("\n"+yaml.safe_dump(descriptor_dict, indent=4, default_flow_style=False))
 755
 756             if (
 757                 desc_type == "vnf"
 758                 and (
 759                     "vnfd:vnfd-catalog" in descriptor_dict
 760                     or "vnfd-catalog" in descriptor_dict
 761                 )
 762             ) or (
 763                 desc_type == "ns"
 764                 and (
 765                     "nsd:nsd-catalog" in descriptor_dict
 766                     or "nsd-catalog" in descriptor_dict
 767                 )
 768             ):
 769                 charms_set = self._charms_search_on_osm_im_dict(
 770                     descriptor_dict, desc_type
 771                 )
 772             else:
 773                 if desc_type == "ns":
 774                     get_charm_list = self._charms_search_on_nsd_sol006_dict
 775                 elif desc_type == "vnf":
 776                     get_charm_list = self._charms_search_on_vnfd_sol006_dict
 777                 else:
 778                     raise Exception("Bad descriptor type")
 779                 charms_set = get_charm_list(descriptor_dict)
 780         return charms_set
 781
 782     def _charms_search_on_osm_im_dict(self, osm_im_dict, desc_type):
 783         self._logger.debug("")
 784         charms_set = []
 785         for k1, v1 in osm_im_dict.items():
 786             for k2, v2 in v1.items():
 787                 for entry in v2:
 788                     if "{}-configuration".format(desc_type) in entry:
 789                         vnf_config = entry["{}-configuration".format(desc_type)]
 790                         for k3, v3 in vnf_config.items():
 791                             if "charm" in v3:
 792                                 charms_set.add((v3["charm"]))
 793                     if "vdu" in entry:
 794                         vdus = entry["vdu"]
 795                         for vdu in vdus:
 796                             if "vdu-configuration" in vdu:
 797                                 for k4, v4 in vdu["vdu-configuration"].items():
 798                                     if "charm" in v4:
 799                                         charms_set.add((v4["charm"]))
 800         return charms_set
 801
 802     def _charms_search_on_vnfd_sol006_dict(self, sol006_dict):
 803         self._logger.debug("")
 804         charms_set = set()
 805         dfs = sol006_dict.get("vnfd", {}).get("df", [])
 806         for df in dfs:
 807             day_1_2s = (
 808                 df.get("lcm-operations-configuration", {})
 809                 .get("operate-vnf-op-config", {})
 810                 .get("day1-2")
 811             )
 812             if day_1_2s is not None:
 813                 for day_1_2 in day_1_2s:
 814                     exec_env_list = day_1_2.get("execution-environment-list", [])
 815                     for exec_env in exec_env_list:
 816                         if "juju" in exec_env and "charm" in exec_env["juju"]:
 817                             charms_set.add(exec_env["juju"]["charm"])
 818         return charms_set
 819
 820     def _charms_search_on_nsd_sol006_dict(self, sol006_dict):
 821         self._logger.debug("")
 822         charms_set = set()
 823         nsd_list = sol006_dict.get("nsd", {}).get("nsd", [])
 824         for nsd in nsd_list:
 825             charm = nsd.get("ns-configuration", {}).get("juju", {}).get("charm")
 826             if charm:
 827                 charms_set.add(charm)
 828         return charms_set