osmclient/common/package_tool.py

   1 # /bin/env python3
   2 # Copyright 2019 ATOS
   3 #
   4 # All Rights Reserved.
   5 #
   6 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 #    not use this file except in compliance with the License. You may obtain
   8 #    a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #    Unless required by applicable law or agreed to in writing, software
  13 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 #    License for the specific language governing permissions and limitations
  16 #    under the License.
  17
  18 import glob
  19 import hashlib
  20 import logging
  21 import os
  22 import shutil
  23 import subprocess
  24 import tarfile
  25 import time
  26 from jinja2 import Environment, PackageLoader
  27 from osm_im.validation import Validation as validation_im
  28 from osm_im.validation import ValidationException
  29 from osm_im import im_translation
  30 from osmclient.common import package_handling as package_handling
  31 from osmclient.common.exceptions import ClientException
  32 from .sol004_package import SOL004Package
  33 from .sol007_package import SOL007Package
  34 import yaml
  35
  36
  37 class PackageTool(object):
  38     def __init__(self, client=None):
  39         self._client = client
  40         self._logger = logging.getLogger("osmclient")
  41         self._validator = validation_im()
  42
  43     def create(
  44         self,
  45         package_type,
  46         base_directory,
  47         package_name,
  48         override,
  49         image,
  50         vdus,
  51         vcpu,
  52         memory,
  53         storage,
  54         interfaces,
  55         vendor,
  56         detailed,
  57         netslice_subnets,
  58         netslice_vlds,
  59         old,
  60     ):
  61         """
  62         **Create a package descriptor**
  63
  64         :params:
  65             - package_type: [vnf, ns, nst]
  66             - base directory: path of destination folder
  67             - package_name: is the name of the package to be created
  68             - image: specify the image of the vdu
  69             - vcpu: number of virtual cpus of the vdu
  70             - memory: amount of memory in MB pf the vdu
  71             - storage: amount of storage in GB of the vdu
  72             - interfaces: number of interfaces besides management interface
  73             - vendor: vendor name of the vnf/ns
  74             - detailed: include all possible values for NSD, VNFD, NST
  75             - netslice_subnets: number of netslice_subnets for the NST
  76             - netslice_vlds: number of virtual link descriptors for the NST
  77             - old: flag to create a descriptor using the previous OSM format (pre SOL006, OSM<9)
  78
  79         :return: status
  80         """
  81         self._logger.debug("")
  82         # print("location: {}".format(osmclient.__path__))
  83         file_loader = PackageLoader("osmclient")
  84         env = Environment(loader=file_loader)
  85         if package_type == "ns":
  86             template = env.get_template("nsd.yaml.j2" if not old else "nsd_old.yaml.j2")
  87             content = {
  88                 "name": package_name,
  89                 "vendor": vendor,
  90                 "vdus": vdus,
  91                 "clean": False,
  92                 "interfaces": interfaces,
  93                 "detailed": detailed,
  94             }
  95         elif package_type == "vnf":
  96             template = env.get_template(
  97                 "vnfd.yaml.j2" if not old else "vnfd_old.yaml.j2"
  98             )
  99             content = {
 100                 "name": package_name,
 101                 "vendor": vendor,
 102                 "vdus": vdus,
 103                 "clean": False,
 104                 "interfaces": interfaces,
 105                 "image": image,
 106                 "vcpu": vcpu,
 107                 "memory": memory,
 108                 "storage": storage,
 109                 "detailed": detailed,
 110             }
 111         elif package_type == "nst":
 112             # TODO: repo-index did not support nst in OSM<9, no changes in template
 113             template = env.get_template("nst.yaml.j2")
 114             content = {
 115                 "name": package_name,
 116                 "vendor": vendor,
 117                 "interfaces": interfaces,
 118                 "netslice_subnets": netslice_subnets,
 119                 "netslice_vlds": netslice_vlds,
 120                 "detailed": detailed,
 121             }
 122         else:
 123             raise ClientException(
 124                 "Wrong descriptor type {}. Options: ns, vnf, nst".format(package_type)
 125             )
 126
 127         self._logger.debug("To be rendered: {}".format(content))
 128         output = template.render(content)
 129         self._logger.debug(output)
 130
 131         structure = self.discover_folder_structure(
 132             base_directory, package_name, override
 133         )
 134         if structure.get("folders"):
 135             self.create_folders(structure["folders"], package_type)
 136         if structure.get("files"):
 137             self.create_files(structure["files"], output, package_type)
 138         return "Created"
 139
 140     def validate(self, base_directory, recursive=True, old_format=False):
 141         """
 142         **Validate OSM Descriptors given a path**
 143
 144         :params:
 145             - base_directory is the root path for all descriptors
 146
 147         :return: List of dict of validated descriptors. keys: type, path, valid, error
 148         """
 149         self._logger.debug("")
 150         table = []
 151         if recursive:
 152             descriptors_paths = [
 153                 f for f in glob.glob(base_directory + "/**/*.yaml", recursive=recursive)
 154             ]
 155         else:
 156             descriptors_paths = [
 157                 f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)
 158             ]
 159         self._logger.info("Base directory: {}".format(base_directory))
 160         self._logger.info(
 161             "{} Descriptors found to validate".format(len(descriptors_paths))
 162         )
 163         for desc_path in descriptors_paths:
 164             with open(desc_path) as descriptor_file:
 165                 descriptor_data = descriptor_file.read()
 166             desc_type = "-"
 167             try:
 168                 # TODO: refactor validation_im.yaml_validation to @staticmethod
 169                 desc_type, descriptor_data = validation_im.yaml_validation(
 170                     self, descriptor_data
 171                 )
 172                 self._logger.debug(f"Validate {desc_type} {descriptor_data}")
 173                 if not old_format:
 174                     if desc_type == "vnfd" or desc_type == "nsd":
 175                         self._logger.error(
 176                             "OSM descriptor '{}' written in an unsupported format. Please update to ETSI SOL006 format".format(
 177                                 desc_path
 178                             )
 179                         )
 180                         self._logger.warning(
 181                             "Package validation skipped. It can still be done with 'osm package-validate --old'"
 182                         )
 183                         self._logger.warning(
 184                             "Package build can still be done with 'osm package-build --skip-validation'"
 185                         )
 186                         raise Exception("Not SOL006 format")
 187                 validation_im.pyangbind_validation(self, desc_type, descriptor_data)
 188                 table.append(
 189                     {"type": desc_type, "path": desc_path, "valid": "OK", "error": "-"}
 190                 )
 191             except Exception as e:
 192                 self._logger.error(f"Validation error: {e}", exc_info=True)
 193                 table.append(
 194                     {
 195                         "type": desc_type,
 196                         "path": desc_path,
 197                         "valid": "ERROR",
 198                         "error": str(e),
 199                     }
 200                 )
 201             self._logger.debug(table[-1])
 202         return table
 203
 204     def translate(self, base_directory, recursive=True, dryrun=False):
 205         """
 206         **Translate OSM Packages given a path**
 207
 208         :params:
 209             - base_directory is the root path for all packages
 210
 211         :return: List of dict of translated packages. keys: current type, new type, path, valid, translated, error
 212         """
 213         self._logger.debug("")
 214         table = []
 215         if recursive:
 216             descriptors_paths = [
 217                 f for f in glob.glob(base_directory + "/**/*.yaml", recursive=recursive)
 218             ]
 219         else:
 220             descriptors_paths = [
 221                 f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)
 222             ]
 223         print("Base directory: {}".format(base_directory))
 224         print("{} Descriptors found to validate".format(len(descriptors_paths)))
 225         for desc_path in descriptors_paths:
 226             with open(desc_path) as descriptor_file:
 227                 descriptor_data = descriptor_file.read()
 228             desc_type = "-"
 229             try:
 230                 desc_type, descriptor_data = validation_im.yaml_validation(
 231                     self, descriptor_data
 232                 )
 233                 self._logger.debug("desc_type: {}".format(desc_type))
 234                 self._logger.debug("descriptor_data:\n{}".format(descriptor_data))
 235                 self._validator.pyangbind_validation(desc_type, descriptor_data)
 236                 if not (desc_type == "vnfd" or desc_type == "nsd"):
 237                     table.append(
 238                         {
 239                             "current type": desc_type,
 240                             "new type": desc_type,
 241                             "path": desc_path,
 242                             "valid": "OK",
 243                             "translated": "N/A",
 244                             "error": "-",
 245                         }
 246                     )
 247                 else:
 248                     new_desc_type = desc_type
 249                     try:
 250                         sol006_model = yaml.safe_dump(
 251                             im_translation.translate_im_model_to_sol006(
 252                                 descriptor_data
 253                             ),
 254                             indent=4,
 255                             default_flow_style=False,
 256                         )
 257                         (
 258                             new_desc_type,
 259                             new_descriptor_data,
 260                         ) = self._validator.yaml_validation(sol006_model)
 261                         self._validator.pyangbind_validation(
 262                             new_desc_type, new_descriptor_data
 263                         )
 264                         if not dryrun:
 265                             with open(desc_path, "w") as descriptor_file:
 266                                 descriptor_file.write(sol006_model)
 267                         table.append(
 268                             {
 269                                 "current type": desc_type,
 270                                 "new type": new_desc_type,
 271                                 "path": desc_path,
 272                                 "valid": "OK",
 273                                 "translated": "OK",
 274                                 "error": "-",
 275                             }
 276                         )
 277                     except ValidationException as ve2:
 278                         table.append(
 279                             {
 280                                 "current type": desc_type,
 281                                 "new type": new_desc_type,
 282                                 "path": desc_path,
 283                                 "valid": "OK",
 284                                 "translated": "ERROR",
 285                                 "error": "Error in the post-validation: {}".format(
 286                                     str(ve2)
 287                                 ),
 288                             }
 289                         )
 290                     except Exception as e2:
 291                         table.append(
 292                             {
 293                                 "current type": desc_type,
 294                                 "new type": new_desc_type,
 295                                 "path": desc_path,
 296                                 "valid": "OK",
 297                                 "translated": "ERROR",
 298                                 "error": "Error in the translation: {}".format(str(e2)),
 299                             }
 300                         )
 301             except ValidationException as ve:
 302                 table.append(
 303                     {
 304                         "current type": desc_type,
 305                         "new type": "N/A",
 306                         "path": desc_path,
 307                         "valid": "ERROR",
 308                         "translated": "N/A",
 309                         "error": "Error in the pre-validation: {}".format(str(ve)),
 310                     }
 311                 )
 312             except Exception as e:
 313                 table.append(
 314                     {
 315                         "current type": desc_type,
 316                         "new type": "N/A",
 317                         "path": desc_path,
 318                         "valid": "ERROR",
 319                         "translated": "N/A",
 320                         "error": str(e),
 321                     }
 322                 )
 323         return table
 324
 325     def descriptor_translate(self, descriptor_file):
 326         """
 327         **Translate input descriptor file from Rel EIGHT OSM to SOL006**
 328
 329         :params:
 330             - base_directory is the root path for all packages
 331
 332         :return: YAML descriptor in the new format
 333         """
 334         self._logger.debug("")
 335         with open(descriptor_file, "r") as df:
 336             im_model = yaml.safe_load(df.read())
 337         sol006_model = im_translation.translate_im_model_to_sol006(im_model)
 338         return yaml.safe_dump(sol006_model, indent=4, default_flow_style=False)
 339
 340     def build(self, package_folder, skip_validation=False, skip_charm_build=False):
 341         """
 342         **Creates a .tar.gz file given a package_folder**
 343
 344         :params:
 345             - package_folder: is the name of the folder to be packaged
 346             - skip_validation: is the flag to validate or not the descriptors on the folder before build
 347
 348         :returns: message result for the build process
 349         """
 350         self._logger.debug("")
 351         package_folder = package_folder.rstrip("/")
 352         if not os.path.exists("{}".format(package_folder)):
 353             return "Fail, package is not in the specified path"
 354         if not skip_validation:
 355             print("Validating package {}".format(package_folder))
 356             results = self.validate(package_folder, recursive=False)
 357             if results:
 358                 for result in results:
 359                     if result["valid"] != "OK":
 360                         raise ClientException(
 361                             "There was an error validating the file {} with error: {}".format(
 362                                 result["path"], result["error"]
 363                             )
 364                         )
 365                 print("Validation OK")
 366             else:
 367                 raise ClientException(
 368                     "No descriptor file found in: {}".format(package_folder)
 369                 )
 370
 371         is_sol004_007 = (
 372             package_handling.get_package_type(package_folder)
 373             != package_handling.OSM_OLD
 374         )
 375
 376         charm_list = self.build_all_charms(
 377             package_folder, skip_charm_build, is_sol004_007
 378         )
 379         return self.build_compressed_file(package_folder, charm_list, is_sol004_007)
 380
 381     def calculate_checksum(self, package_folder):
 382         """
 383         **Function to calculate the checksum given a folder**
 384
 385         :params:
 386             - package_folder: is the folder where we have the files to calculate the checksum
 387         :returns: None
 388         """
 389         self._logger.debug("")
 390         files = [
 391             f
 392             for f in glob.glob(package_folder + "/**/*.*", recursive=True)
 393             if os.path.isfile(f)
 394         ]
 395         with open("{}/checksums.txt".format(package_folder), "w+") as checksum:
 396             for file_item in files:
 397                 if "checksums.txt" in file_item:
 398                     continue
 399                 # from https://www.quickprogrammingtips.com/python/how-to-calculate-md5-hash-of-a-file-in-python.html
 400                 md5_hash = hashlib.md5()
 401                 with open(file_item, "rb") as f:
 402                     # Read and update hash in chunks of 4K
 403                     for byte_block in iter(lambda: f.read(4096), b""):
 404                         md5_hash.update(byte_block)
 405                     checksum.write("{}\t{}\n".format(md5_hash.hexdigest(), file_item))
 406
 407     def create_folders(self, folders, package_type):
 408         """
 409         **Create folder given a list of folders**
 410
 411         :params:
 412             - folders: [List] list of folders paths to be created
 413             - package_type: is the type of package to be created
 414         :return: None
 415         """
 416         self._logger.debug("")
 417         for folder in folders:
 418             try:
 419                 # print("Folder {} == package_type {}".format(folder[1], package_type))
 420                 if folder[1] == package_type:
 421                     print("Creating folder:\t{}".format(folder[0]))
 422                     os.makedirs(folder[0])
 423             except FileExistsError:
 424                 pass
 425
 426     def save_file(self, file_name, file_body):
 427         """
 428         **Create a file given a name and the content**
 429
 430         :params:
 431             - file_name: is the name of the file with the relative route
 432             - file_body: is the content of the file
 433         :return: None
 434         """
 435         self._logger.debug("")
 436         print("Creating file:  \t{}".format(file_name))
 437         try:
 438             with open(file_name, "w+") as f:
 439                 f.write(file_body)
 440         except Exception as e:
 441             raise ClientException(e)
 442
 443     def generate_readme(self):
 444         """
 445         **Creates the README content**
 446
 447         :returns: readme content
 448         """
 449         self._logger.debug("")
 450         return """# Descriptor created by OSM descriptor package generated\n\n**Created on {} **""".format(
 451             time.strftime("%m/%d/%Y, %H:%M:%S", time.localtime())
 452         )
 453
 454     def generate_cloud_init(self):
 455         """
 456         **Creates the cloud-init content**
 457
 458         :returns: cloud-init content
 459         """
 460         self._logger.debug("")
 461         return "---\n#cloud-config"
 462
 463     def create_files(self, files, file_content, package_type):
 464         """
 465         **Creates the files given the file list and type**
 466
 467         :params:
 468             - files: is the list of files structure
 469             - file_content: is the content of the descriptor rendered by the template
 470             - package_type: is the type of package to filter the creation structure
 471
 472         :return: None
 473         """
 474         self._logger.debug("")
 475         for file_item, file_package, file_type in files:
 476             if package_type == file_package:
 477                 if file_type == "descriptor":
 478                     self.save_file(file_item, file_content)
 479                 elif file_type == "readme":
 480                     self.save_file(file_item, self.generate_readme())
 481                 elif file_type == "cloud_init":
 482                     self.save_file(file_item, self.generate_cloud_init())
 483
 484     def check_files_folders(self, path_list, override):
 485         """
 486         **Find files and folders missing given a directory structure {"folders": [], "files": []}**
 487
 488         :params:
 489             - path_list: is the list of files and folders to be created
 490             - override: is the flag used to indicate the creation of the list even if the file exist to override it
 491
 492         :return: Missing paths Dict
 493         """
 494         self._logger.debug("")
 495         missing_paths = {}
 496         folders = []
 497         files = []
 498         for folder in path_list.get("folders"):
 499             if not os.path.exists(folder[0]):
 500                 folders.append(folder)
 501         missing_paths["folders"] = folders
 502
 503         for file_item in path_list.get("files"):
 504             if not os.path.exists(file_item[0]) or override is True:
 505                 files.append(file_item)
 506         missing_paths["files"] = files
 507
 508         return missing_paths
 509
 510     def build_all_charms(self, package_folder, skip_charm_build, sol004_007=True):
 511         """
 512         **Read the descriptor file, check that the charms referenced are in the folder and compiles them**
 513
 514         :params:
 515             - packet_folder: is the location of the package
 516         :return: Files and Folders not found. In case of override, it will return all file list
 517         """
 518         self._logger.debug("")
 519         charms_set = set()
 520         descriptor_file = False
 521         package_type = package_handling.get_package_type(package_folder)
 522         if sol004_007 and package_type.find("TOSCA") >= 0:
 523             descriptors_paths = [
 524                 f for f in glob.glob(package_folder + "/Definitions/*.yaml")
 525             ]
 526         else:
 527             descriptors_paths = [f for f in glob.glob(package_folder + "/*.yaml")]
 528         for file in descriptors_paths:
 529             if file.endswith("nfd.yaml"):
 530                 descriptor_file = True
 531                 charms_set = self.charms_search(file, "vnf")
 532             if file.endswith("nsd.yaml"):
 533                 descriptor_file = True
 534                 charms_set = self.charms_search(file, "ns")
 535         print("List of charms in the descriptor: {}".format(charms_set))
 536         if not descriptor_file:
 537             raise ClientException(
 538                 'Descriptor filename is not correct in: {}. It should end with "nfd.yaml" or "nsd.yaml"'.format(
 539                     package_folder
 540                 )
 541             )
 542         if charms_set and not skip_charm_build:
 543             for charmName in charms_set:
 544                 if os.path.isdir(
 545                     "{}/{}charms/layers/{}".format(
 546                         package_folder, "Scripts/" if sol004_007 else "", charmName
 547                     )
 548                 ):
 549                     print(
 550                         "Building charm {}/{}charms/layers/{}".format(
 551                             package_folder, "Scripts/" if sol004_007 else "", charmName
 552                         )
 553                     )
 554                     self.charm_build(package_folder, charmName, sol004_007)
 555                     print("Charm built: {}".format(charmName))
 556                 elif os.path.isdir(
 557                     "{}/{}charms/ops/{}".format(
 558                         package_folder, "Scripts/" if sol004_007 else "", charmName
 559                     )
 560                 ):
 561                     self.charmcraft_build(package_folder, charmName)
 562                 else:
 563                     if not os.path.isdir(
 564                         "{}/{}charms/{}".format(
 565                             package_folder, "Scripts/" if sol004_007 else "", charmName
 566                         )
 567                     ) and not os.path.isfile(
 568                         "{}/{}charms/{}".format(
 569                             package_folder, "Scripts/" if sol004_007 else "", charmName
 570                         )
 571                     ):
 572                         raise ClientException(
 573                             "The charm: {} referenced in the descriptor file "
 574                             "is not present either in {}/charms or in {}/charms/layers".format(
 575                                 charmName, package_folder, package_folder
 576                             )
 577                         )
 578         self._logger.debug("Return list of charms: {}".format(charms_set))
 579         return charms_set
 580
 581     def discover_folder_structure(self, base_directory, name, override):
 582         """
 583         **Discover files and folders structure for SOL004/SOL007 descriptors given a base_directory and name**
 584
 585         :params:
 586             - base_directory: is the location of the package to be created
 587             - name: is the name of the package
 588             - override: is the flag used to indicate the creation of the list even if the file exist to override it
 589         :return: Files and Folders not found. In case of override, it will return all file list
 590         """
 591         self._logger.debug("")
 592         prefix = "{}/{}".format(base_directory, name)
 593         files_folders = {
 594             "folders": [
 595                 ("{}_ns".format(prefix), "ns"),
 596                 ("{}_ns/Licenses".format(prefix), "ns"),
 597                 ("{}_ns/Files/icons".format(prefix), "ns"),
 598                 ("{}_ns/Scripts/charms".format(prefix), "ns"),
 599                 ("{}_vnf".format(name), "vnf"),
 600                 ("{}_vnf/Licenses".format(prefix), "vnf"),
 601                 ("{}_vnf/Scripts/charms".format(prefix), "vnf"),
 602                 ("{}_vnf/Scripts/cloud_init".format(prefix), "vnf"),
 603                 ("{}_vnf/Files/images".format(prefix), "vnf"),
 604                 ("{}_vnf/Files/icons".format(prefix), "vnf"),
 605                 ("{}_vnf/Scripts/scripts".format(prefix), "vnf"),
 606                 ("{}_nst".format(prefix), "nst"),
 607                 ("{}_nst/icons".format(prefix), "nst"),
 608             ],
 609             "files": [
 610                 ("{}_ns/{}_nsd.yaml".format(prefix, name), "ns", "descriptor"),
 611                 ("{}_ns/README.md".format(prefix), "ns", "readme"),
 612                 ("{}_vnf/{}_vnfd.yaml".format(prefix, name), "vnf", "descriptor"),
 613                 (
 614                     "{}_vnf/Scripts/cloud_init/cloud-config.txt".format(prefix),
 615                     "vnf",
 616                     "cloud_init",
 617                 ),
 618                 ("{}_vnf/README.md".format(prefix), "vnf", "readme"),
 619                 ("{}_nst/{}_nst.yaml".format(prefix, name), "nst", "descriptor"),
 620                 ("{}_nst/README.md".format(prefix), "nst", "readme"),
 621             ],
 622         }
 623         missing_files_folders = self.check_files_folders(files_folders, override)
 624         # print("Missing files and folders: {}".format(missing_files_folders))
 625         return missing_files_folders
 626
 627     def charm_build(self, charms_folder, build_name, sol004_007=True):
 628         """
 629         Build the charms inside the package.
 630         params: package_folder is the name of the folder where is the charms to compile.
 631                 build_name is the name of the layer or interface
 632         """
 633         self._logger.debug("")
 634
 635         if sol004_007:
 636             os.environ["JUJU_REPOSITORY"] = "{}/Scripts/charms".format(charms_folder)
 637         else:
 638             os.environ["JUJU_REPOSITORY"] = "{}/charms".format(charms_folder)
 639
 640         os.environ["CHARM_LAYERS_DIR"] = "{}/layers".format(
 641             os.environ["JUJU_REPOSITORY"]
 642         )
 643         os.environ["CHARM_INTERFACES_DIR"] = "{}/interfaces".format(
 644             os.environ["JUJU_REPOSITORY"]
 645         )
 646
 647         if sol004_007:
 648             os.environ["CHARM_BUILD_DIR"] = "{}/Scripts/charms/builds".format(
 649                 charms_folder
 650             )
 651         else:
 652             os.environ["CHARM_BUILD_DIR"] = "{}/charms/builds".format(charms_folder)
 653
 654         if not os.path.exists(os.environ["CHARM_BUILD_DIR"]):
 655             os.makedirs(os.environ["CHARM_BUILD_DIR"])
 656         src_folder = "{}/{}".format(os.environ["CHARM_LAYERS_DIR"], build_name)
 657         result = subprocess.run(["charm", "build", "{}".format(src_folder)])
 658         if result.returncode == 1:
 659             raise ClientException("failed to build the charm: {}".format(src_folder))
 660         self._logger.verbose("charm {} built".format(src_folder))
 661
 662     def charmcraft_build(self, package_folder, charm_name):
 663         """
 664         Build the charms inside the package (new operator framework charms)
 665         params: package_folder is the name of the folder where is the charms to compile.
 666                 build_name is the name of the layer or interface
 667         """
 668         self._logger.debug("Building charm {}".format(charm_name))
 669         src_folder = f"{package_folder}/Scripts/charms/ops/{charm_name}"
 670         current_directory = os.getcwd()
 671         os.chdir(src_folder)
 672         try:
 673             result = subprocess.run(["charmcraft", "build"])
 674             if result.returncode == 1:
 675                 raise ClientException(
 676                     "failed to build the charm: {}".format(src_folder)
 677                 )
 678             subprocess.run(["rm", "-rf", f"../../{charm_name}"])
 679             subprocess.run(["mv", "build", f"../../{charm_name}"])
 680             self._logger.verbose("charm {} built".format(src_folder))
 681         finally:
 682             os.chdir(current_directory)
 683
 684     def build_compressed_file(self, package_folder, charm_list=None, sol004_007=True):
 685         if sol004_007:
 686             return self.build_zipfile(package_folder, charm_list)
 687         else:
 688             return self.build_tarfile(package_folder, charm_list)
 689
 690     def build_zipfile(self, package_folder, charm_list=None):
 691         """
 692         Creates a zip file given a package_folder
 693         params: package_folder is the name of the folder to be packaged
 694         returns: .zip name
 695         """
 696         self._logger.debug("")
 697         cwd = None
 698         try:
 699             directory_name, package_name = self.create_temp_dir_sol004_007(
 700                 package_folder, charm_list
 701             )
 702             cwd = os.getcwd()
 703             os.chdir(directory_name)
 704             package_type = package_handling.get_package_type(package_folder)
 705             print(package_type)
 706
 707             if (
 708                 package_handling.SOL007 == package_type
 709                 or package_handling.SOL007_TOSCA == package_type
 710             ):
 711                 the_package = SOL007Package(package_folder)
 712             elif (
 713                 package_handling.SOL004 == package_type
 714                 or package_handling.SOL004_TOSCA == package_type
 715             ):
 716                 the_package = SOL004Package(package_folder)
 717
 718             the_package.create_or_update_metadata_file()
 719
 720             the_zip_package = shutil.make_archive(
 721                 os.path.join(cwd, package_name),
 722                 "zip",
 723                 os.path.join(directory_name, package_name),
 724             )
 725
 726             print("Package created: {}".format(the_zip_package))
 727
 728             return the_zip_package
 729
 730         except Exception as exc:
 731             raise ClientException(
 732                 "failure during build of zip file (create temp dir, calculate checksum, "
 733                 "zip file): {}".format(exc)
 734             )
 735         finally:
 736             if cwd:
 737                 os.chdir(cwd)
 738             shutil.rmtree(os.path.join(package_folder, "tmp"))
 739
 740     def build_tarfile(self, package_folder, charm_list=None):
 741         """
 742         Creates a .tar.gz file given a package_folder
 743         params: package_folder is the name of the folder to be packaged
 744         returns: .tar.gz name
 745         """
 746         self._logger.debug("")
 747         cwd = None
 748         try:
 749             directory_name, package_name = self.create_temp_dir(
 750                 package_folder, charm_list
 751             )
 752             cwd = os.getcwd()
 753             os.chdir(directory_name)
 754             self.calculate_checksum(package_name)
 755             with tarfile.open("{}.tar.gz".format(package_name), mode="w:gz") as archive:
 756                 print("Adding File: {}".format(package_name))
 757                 archive.add("{}".format(package_name), recursive=True)
 758             # return "Created {}.tar.gz".format(package_folder)
 759             # self.build("{}".format(os.path.basename(package_folder)))
 760             os.chdir(cwd)
 761             cwd = None
 762             created_package = "{}/{}.tar.gz".format(
 763                 os.path.dirname(package_folder) or ".", package_name
 764             )
 765             os.rename(
 766                 "{}/{}.tar.gz".format(directory_name, package_name), created_package
 767             )
 768             os.rename(
 769                 "{}/{}/checksums.txt".format(directory_name, package_name),
 770                 "{}/checksums.txt".format(package_folder),
 771             )
 772             print("Package created: {}".format(created_package))
 773             return created_package
 774         except Exception as exc:
 775             raise ClientException(
 776                 "failure during build of targz file (create temp dir, calculate checksum, "
 777                 "tar.gz file): {}".format(exc)
 778             )
 779         finally:
 780             if cwd:
 781                 os.chdir(cwd)
 782             shutil.rmtree(os.path.join(package_folder, "tmp"))
 783
 784     def create_temp_dir(self, package_folder, charm_list=None):
 785         """
 786         Method to create a temporary folder where we can move the files in package_folder
 787         """
 788         self._logger.debug("")
 789         ignore_patterns = ".gitignore"
 790         ignore = shutil.ignore_patterns(ignore_patterns)
 791         directory_name = os.path.abspath(package_folder)
 792         package_name = os.path.basename(directory_name)
 793         directory_name += "/tmp"
 794         os.makedirs("{}/{}".format(directory_name, package_name), exist_ok=True)
 795         self._logger.debug("Makedirs DONE: {}/{}".format(directory_name, package_name))
 796         for item in os.listdir(package_folder):
 797             self._logger.debug("Item: {}".format(item))
 798             if item != "tmp":
 799                 s = os.path.join(package_folder, item)
 800                 d = os.path.join(os.path.join(directory_name, package_name), item)
 801                 if os.path.isdir(s):
 802                     if item == "charms":
 803                         os.makedirs(d, exist_ok=True)
 804                         s_builds = os.path.join(s, "builds")
 805                         for charm in charm_list:
 806                             self._logger.debug("Copying charm {}".format(charm))
 807                             if charm in os.listdir(s):
 808                                 s_charm = os.path.join(s, charm)
 809                             elif charm in os.listdir(s_builds):
 810                                 s_charm = os.path.join(s_builds, charm)
 811                             else:
 812                                 raise ClientException(
 813                                     "The charm {} referenced in the descriptor file "
 814                                     "could not be found in {}/charms or in {}/charms/builds".format(
 815                                         charm, package_folder, package_folder
 816                                     )
 817                                 )
 818                             d_temp = os.path.join(d, charm)
 819                             self._logger.debug(
 820                                 "Copying tree: {} -> {}".format(s_charm, d_temp)
 821                             )
 822                             if os.path.isdir(s_charm):
 823                                 shutil.copytree(
 824                                     s_charm, d_temp, symlinks=True, ignore=ignore
 825                                 )
 826                             else:
 827                                 shutil.copy2(s_charm, d_temp)
 828                             self._logger.debug("DONE")
 829                     else:
 830                         self._logger.debug("Copying tree: {} -> {}".format(s, d))
 831                         shutil.copytree(s, d, symlinks=True, ignore=ignore)
 832                         self._logger.debug("DONE")
 833                 else:
 834                     if item in ignore_patterns:
 835                         continue
 836                     self._logger.debug("Copying file: {} -> {}".format(s, d))
 837                     shutil.copy2(s, d)
 838                     self._logger.debug("DONE")
 839         return directory_name, package_name
 840
 841     def copy_tree(self, s, d, ignore):
 842         self._logger.debug("Copying tree: {} -> {}".format(s, d))
 843         shutil.copytree(s, d, symlinks=True, ignore=ignore)
 844         self._logger.debug("DONE")
 845
 846     def create_temp_dir_sol004_007(self, package_folder, charm_list=None):
 847         """
 848         Method to create a temporary folder where we can move the files in package_folder
 849         """
 850         self._logger.debug("")
 851         ignore_patterns = ".gitignore"
 852         ignore = shutil.ignore_patterns(ignore_patterns)
 853         directory_name = os.path.abspath(package_folder)
 854         package_name = os.path.basename(directory_name)
 855         directory_name += "/tmp"
 856         os.makedirs("{}/{}".format(directory_name, package_name), exist_ok=True)
 857         self._logger.debug("Makedirs DONE: {}/{}".format(directory_name, package_name))
 858         for item in os.listdir(package_folder):
 859             self._logger.debug("Item: {}".format(item))
 860             if item != "tmp":
 861                 s = os.path.join(package_folder, item)
 862                 d = os.path.join(os.path.join(directory_name, package_name), item)
 863                 if os.path.isdir(s):
 864                     if item == "Scripts":
 865                         os.makedirs(d, exist_ok=True)
 866                         scripts_folder = s
 867                         for script_item in os.listdir(scripts_folder):
 868                             scripts_destination_folder = os.path.join(d, script_item)
 869                             if script_item == "charms":
 870                                 s_builds = os.path.join(
 871                                     scripts_folder, script_item, "builds"
 872                                 )
 873                                 for charm in charm_list:
 874                                     self._logger.debug("Copying charm {}".format(charm))
 875                                     if charm in os.listdir(
 876                                         os.path.join(scripts_folder, script_item)
 877                                     ):
 878                                         s_charm = os.path.join(
 879                                             scripts_folder, script_item, charm
 880                                         )
 881                                     elif charm in os.listdir(s_builds):
 882                                         s_charm = os.path.join(s_builds, charm)
 883                                     else:
 884                                         raise ClientException(
 885                                             "The charm {} referenced in the descriptor file "
 886                                             "could not be found in {}/charms or in {}/charms/builds".format(
 887                                                 charm, package_folder, package_folder
 888                                             )
 889                                         )
 890                                     d_temp = os.path.join(
 891                                         scripts_destination_folder, charm
 892                                     )
 893                                     self.copy_tree(s_charm, d_temp, ignore)
 894                             else:
 895                                 self.copy_tree(
 896                                     os.path.join(scripts_folder, script_item),
 897                                     scripts_destination_folder,
 898                                     ignore,
 899                                 )
 900                     else:
 901                         self.copy_tree(s, d, ignore)
 902                 else:
 903                     if item in ignore_patterns:
 904                         continue
 905                     self._logger.debug("Copying file: {} -> {}".format(s, d))
 906                     shutil.copy2(s, d)
 907                     self._logger.debug("DONE")
 908         return directory_name, package_name
 909
 910     def charms_search(self, descriptor_file, desc_type):
 911         self._logger.debug(
 912             "descriptor_file: {}, desc_type: {}".format(descriptor_file, desc_type)
 913         )
 914         charms_set = set()
 915         with open("{}".format(descriptor_file)) as yaml_desc:
 916             descriptor_dict = yaml.safe_load(yaml_desc)
 917             # self._logger.debug("\n"+yaml.safe_dump(descriptor_dict, indent=4, default_flow_style=False))
 918
 919             if (
 920                 desc_type == "vnf"
 921                 and (
 922                     "vnfd:vnfd-catalog" in descriptor_dict
 923                     or "vnfd-catalog" in descriptor_dict
 924                 )
 925             ) or (
 926                 desc_type == "ns"
 927                 and (
 928                     "nsd:nsd-catalog" in descriptor_dict
 929                     or "nsd-catalog" in descriptor_dict
 930                 )
 931             ):
 932                 charms_set = self._charms_search_on_osm_im_dict(
 933                     descriptor_dict, desc_type
 934                 )
 935             else:
 936                 if desc_type == "ns":
 937                     get_charm_list = self._charms_search_on_nsd_sol006_dict
 938                 elif desc_type == "vnf":
 939                     get_charm_list = self._charms_search_on_vnfd_sol006_dict
 940                 else:
 941                     raise Exception("Bad descriptor type")
 942                 charms_set = get_charm_list(descriptor_dict)
 943         return charms_set
 944
 945     def _charms_search_on_osm_im_dict(self, osm_im_dict, desc_type):
 946         self._logger.debug("")
 947         charms_set = []
 948         for k1, v1 in osm_im_dict.items():
 949             for k2, v2 in v1.items():
 950                 for entry in v2:
 951                     if "{}-configuration".format(desc_type) in entry:
 952                         vnf_config = entry["{}-configuration".format(desc_type)]
 953                         for k3, v3 in vnf_config.items():
 954                             if "charm" in v3:
 955                                 charms_set.add((v3["charm"]))
 956                     if "vdu" in entry:
 957                         vdus = entry["vdu"]
 958                         for vdu in vdus:
 959                             if "vdu-configuration" in vdu:
 960                                 for k4, v4 in vdu["vdu-configuration"].items():
 961                                     if "charm" in v4:
 962                                         charms_set.add((v4["charm"]))
 963         return charms_set
 964
 965     def _charms_search_on_vnfd_sol006_dict(self, sol006_dict):
 966         self._logger.debug("")
 967         charms_set = set()
 968         dfs = sol006_dict.get("vnfd", {}).get("df", [])
 969         for df in dfs:
 970             day_1_2s = (
 971                 df.get("lcm-operations-configuration", {})
 972                 .get("operate-vnf-op-config", {})
 973                 .get("day1-2")
 974             )
 975             if day_1_2s is not None:
 976                 for day_1_2 in day_1_2s:
 977                     exec_env_list = day_1_2.get("execution-environment-list", [])
 978                     for exec_env in exec_env_list:
 979                         if "juju" in exec_env and "charm" in exec_env["juju"]:
 980                             charms_set.add(exec_env["juju"]["charm"])
 981         return charms_set
 982
 983     def _charms_search_on_nsd_sol006_dict(self, sol006_dict):
 984         self._logger.debug("")
 985         charms_set = set()
 986         nsd_list = sol006_dict.get("nsd", {}).get("nsd", [])
 987         for nsd in nsd_list:
 988             charm = nsd.get("ns-configuration", {}).get("juju", {}).get("charm")
 989             if charm:
 990                 charms_set.add(charm)
 991         return charms_set