osmclient/common/package_tool.py

   1 # /bin/env python3
   2 # Copyright 2019 ATOS
   3 #
   4 # All Rights Reserved.
   5 #
   6 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 #    not use this file except in compliance with the License. You may obtain
   8 #    a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #    Unless required by applicable law or agreed to in writing, software
  13 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 #    License for the specific language governing permissions and limitations
  16 #    under the License.
  17
  18 import glob
  19 import logging
  20 import os
  21 import shutil
  22 import subprocess
  23 import tarfile
  24 import time
  25 from jinja2 import Environment, PackageLoader
  26 from osm_im.validation import Validation as validation_im
  27 from osm_im.validation import ValidationException
  28 from osm_im import im_translation
  29 from osmclient.common import package_handling as package_handling
  30 from osmclient.common.exceptions import ClientException
  31 from osmclient.common import utils
  32 from .sol004_package import SOL004Package
  33 from .sol007_package import SOL007Package
  34 import yaml
  35
  36
  37 class PackageTool(object):
  38     def __init__(self, client=None):
  39         self._client = client
  40         self._logger = logging.getLogger("osmclient")
  41         self._validator = validation_im()
  42
  43     def create(
  44         self,
  45         package_type,
  46         base_directory,
  47         package_name,
  48         override,
  49         image,
  50         vdus,
  51         vcpu,
  52         memory,
  53         storage,
  54         interfaces,
  55         vendor,
  56         detailed,
  57         netslice_subnets,
  58         netslice_vlds,
  59         old,
  60     ):
  61         """
  62         **Create a package descriptor**
  63
  64         :params:
  65             - package_type: [vnf, ns, nst]
  66             - base directory: path of destination folder
  67             - package_name: is the name of the package to be created
  68             - image: specify the image of the vdu
  69             - vcpu: number of virtual cpus of the vdu
  70             - memory: amount of memory in MB pf the vdu
  71             - storage: amount of storage in GB of the vdu
  72             - interfaces: number of interfaces besides management interface
  73             - vendor: vendor name of the vnf/ns
  74             - detailed: include all possible values for NSD, VNFD, NST
  75             - netslice_subnets: number of netslice_subnets for the NST
  76             - netslice_vlds: number of virtual link descriptors for the NST
  77             - old: flag to create a descriptor using the previous OSM format (pre SOL006, OSM<9)
  78
  79         :return: status
  80         """
  81         self._logger.debug("")
  82         # print("location: {}".format(osmclient.__path__))
  83         file_loader = PackageLoader("osmclient")
  84         env = Environment(loader=file_loader, autoescape=True)
  85         if package_type == "ns":
  86             template = env.get_template("nsd.yaml.j2" if not old else "nsd_old.yaml.j2")
  87             content = {
  88                 "name": package_name,
  89                 "vendor": vendor,
  90                 "vdus": vdus,
  91                 "clean": False,
  92                 "interfaces": interfaces,
  93                 "detailed": detailed,
  94             }
  95         elif package_type == "vnf":
  96             template = env.get_template(
  97                 "vnfd.yaml.j2" if not old else "vnfd_old.yaml.j2"
  98             )
  99             content = {
 100                 "name": package_name,
 101                 "vendor": vendor,
 102                 "vdus": vdus,
 103                 "clean": False,
 104                 "interfaces": interfaces,
 105                 "image": image,
 106                 "vcpu": vcpu,
 107                 "memory": memory,
 108                 "storage": storage,
 109                 "detailed": detailed,
 110             }
 111         elif package_type == "nst":
 112             # TODO: repo-index did not support nst in OSM<9, no changes in template
 113             template = env.get_template("nst.yaml.j2")
 114             content = {
 115                 "name": package_name,
 116                 "vendor": vendor,
 117                 "interfaces": interfaces,
 118                 "netslice_subnets": netslice_subnets,
 119                 "netslice_vlds": netslice_vlds,
 120                 "detailed": detailed,
 121             }
 122         else:
 123             raise ClientException(
 124                 "Wrong descriptor type {}. Options: ns, vnf, nst".format(package_type)
 125             )
 126
 127         self._logger.debug("To be rendered: {}".format(content))
 128         output = template.render(content)
 129         self._logger.debug(output)
 130
 131         structure = self.discover_folder_structure(
 132             base_directory, package_name, override
 133         )
 134         if structure.get("folders"):
 135             self.create_folders(structure["folders"], package_type)
 136         if structure.get("files"):
 137             self.create_files(structure["files"], output, package_type)
 138         return "Created"
 139
 140     def validate(self, base_directory, recursive=True, old_format=False):
 141         """
 142         **Validate OSM Descriptors given a path**
 143
 144         :params:
 145             - base_directory is the root path for all descriptors
 146
 147         :return: List of dict of validated descriptors. keys: type, path, valid, error
 148         """
 149         self._logger.debug("")
 150         table = []
 151         if recursive:
 152             descriptors_paths = [
 153                 f for f in glob.glob(base_directory + "/**/*.yaml", recursive=recursive)
 154             ]
 155         else:
 156             descriptors_paths = [
 157                 f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)
 158             ]
 159         self._logger.info("Base directory: {}".format(base_directory))
 160         self._logger.info(
 161             "{} Descriptors found to validate".format(len(descriptors_paths))
 162         )
 163         for desc_path in descriptors_paths:
 164             with open(desc_path) as descriptor_file:
 165                 descriptor_data = descriptor_file.read()
 166             desc_type = "-"
 167             try:
 168                 # TODO: refactor validation_im.yaml_validation to @staticmethod
 169                 desc_type, descriptor_data = validation_im.yaml_validation(
 170                     self, descriptor_data
 171                 )
 172                 self._logger.debug(f"Validate {desc_type} {descriptor_data}")
 173                 if not old_format:
 174                     if desc_type == "vnfd" or desc_type == "nsd":
 175                         self._logger.error(
 176                             "OSM descriptor '{}' written in an unsupported format. Please update to ETSI SOL006 format".format(
 177                                 desc_path
 178                             )
 179                         )
 180                         self._logger.warning(
 181                             "Package validation skipped. It can still be done with 'osm package-validate --old'"
 182                         )
 183                         self._logger.warning(
 184                             "Package build can still be done with 'osm package-build --skip-validation'"
 185                         )
 186                         raise Exception("Not SOL006 format")
 187                 validation_im.pyangbind_validation(self, desc_type, descriptor_data)
 188                 table.append(
 189                     {"type": desc_type, "path": desc_path, "valid": "OK", "error": "-"}
 190                 )
 191             except Exception as e:
 192                 self._logger.error(f"Validation error: {e}", exc_info=True)
 193                 table.append(
 194                     {
 195                         "type": desc_type,
 196                         "path": desc_path,
 197                         "valid": "ERROR",
 198                         "error": str(e),
 199                     }
 200                 )
 201             self._logger.debug(table[-1])
 202         return table
 203
 204     def translate(self, base_directory, recursive=True, dryrun=False):
 205         """
 206         **Translate OSM Packages given a path**
 207
 208         :params:
 209             - base_directory is the root path for all packages
 210
 211         :return: List of dict of translated packages. keys: current type, new type, path, valid, translated, error
 212         """
 213         self._logger.debug("")
 214         table = []
 215         if recursive:
 216             descriptors_paths = [
 217                 f for f in glob.glob(base_directory + "/**/*.yaml", recursive=recursive)
 218             ]
 219         else:
 220             descriptors_paths = [
 221                 f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)
 222             ]
 223         print("Base directory: {}".format(base_directory))
 224         print("{} Descriptors found to validate".format(len(descriptors_paths)))
 225         for desc_path in descriptors_paths:
 226             with open(desc_path) as descriptor_file:
 227                 descriptor_data = descriptor_file.read()
 228             desc_type = "-"
 229             try:
 230                 desc_type, descriptor_data = validation_im.yaml_validation(
 231                     self, descriptor_data
 232                 )
 233                 self._logger.debug("desc_type: {}".format(desc_type))
 234                 self._logger.debug("descriptor_data:\n{}".format(descriptor_data))
 235                 self._validator.pyangbind_validation(desc_type, descriptor_data)
 236                 if not (desc_type == "vnfd" or desc_type == "nsd"):
 237                     table.append(
 238                         {
 239                             "current type": desc_type,
 240                             "new type": desc_type,
 241                             "path": desc_path,
 242                             "valid": "OK",
 243                             "translated": "N/A",
 244                             "error": "-",
 245                         }
 246                     )
 247                 else:
 248                     new_desc_type = desc_type
 249                     try:
 250                         sol006_model = yaml.safe_dump(
 251                             im_translation.translate_im_model_to_sol006(
 252                                 descriptor_data
 253                             ),
 254                             indent=4,
 255                             default_flow_style=False,
 256                         )
 257                         (
 258                             new_desc_type,
 259                             new_descriptor_data,
 260                         ) = self._validator.yaml_validation(sol006_model)
 261                         self._validator.pyangbind_validation(
 262                             new_desc_type, new_descriptor_data
 263                         )
 264                         if not dryrun:
 265                             with open(desc_path, "w") as descriptor_file:
 266                                 descriptor_file.write(sol006_model)
 267                         table.append(
 268                             {
 269                                 "current type": desc_type,
 270                                 "new type": new_desc_type,
 271                                 "path": desc_path,
 272                                 "valid": "OK",
 273                                 "translated": "OK",
 274                                 "error": "-",
 275                             }
 276                         )
 277                     except ValidationException as ve2:
 278                         table.append(
 279                             {
 280                                 "current type": desc_type,
 281                                 "new type": new_desc_type,
 282                                 "path": desc_path,
 283                                 "valid": "OK",
 284                                 "translated": "ERROR",
 285                                 "error": "Error in the post-validation: {}".format(
 286                                     str(ve2)
 287                                 ),
 288                             }
 289                         )
 290                     except Exception as e2:
 291                         table.append(
 292                             {
 293                                 "current type": desc_type,
 294                                 "new type": new_desc_type,
 295                                 "path": desc_path,
 296                                 "valid": "OK",
 297                                 "translated": "ERROR",
 298                                 "error": "Error in the translation: {}".format(str(e2)),
 299                             }
 300                         )
 301             except ValidationException as ve:
 302                 table.append(
 303                     {
 304                         "current type": desc_type,
 305                         "new type": "N/A",
 306                         "path": desc_path,
 307                         "valid": "ERROR",
 308                         "translated": "N/A",
 309                         "error": "Error in the pre-validation: {}".format(str(ve)),
 310                     }
 311                 )
 312             except Exception as e:
 313                 table.append(
 314                     {
 315                         "current type": desc_type,
 316                         "new type": "N/A",
 317                         "path": desc_path,
 318                         "valid": "ERROR",
 319                         "translated": "N/A",
 320                         "error": str(e),
 321                     }
 322                 )
 323         return table
 324
 325     def descriptor_translate(self, descriptor_file):
 326         """
 327         **Translate input descriptor file from Rel EIGHT OSM to SOL006**
 328
 329         :params:
 330             - base_directory is the root path for all packages
 331
 332         :return: YAML descriptor in the new format
 333         """
 334         self._logger.debug("")
 335         with open(descriptor_file, "r") as df:
 336             im_model = yaml.safe_load(df.read())
 337         sol006_model = im_translation.translate_im_model_to_sol006(im_model)
 338         return yaml.safe_dump(sol006_model, indent=4, default_flow_style=False)
 339
 340     def build(self, package_folder, skip_validation=False, skip_charm_build=False):
 341         """
 342         **Creates a .tar.gz file given a package_folder**
 343
 344         :params:
 345             - package_folder: is the name of the folder to be packaged
 346             - skip_validation: is the flag to validate or not the descriptors on the folder before build
 347
 348         :returns: message result for the build process
 349         """
 350         self._logger.debug("")
 351         package_folder = package_folder.rstrip("/")
 352         if not os.path.exists("{}".format(package_folder)):
 353             return "Fail, package is not in the specified path"
 354         if not skip_validation:
 355             print("Validating package {}".format(package_folder))
 356             results = self.validate(package_folder, recursive=False)
 357             if results:
 358                 for result in results:
 359                     if result["valid"] != "OK":
 360                         raise ClientException(
 361                             "There was an error validating the file {} with error: {}".format(
 362                                 result["path"], result["error"]
 363                             )
 364                         )
 365                 print("Validation OK")
 366             else:
 367                 raise ClientException(
 368                     "No descriptor file found in: {}".format(package_folder)
 369                 )
 370
 371         is_sol004_007 = (
 372             package_handling.get_package_type(package_folder)
 373             != package_handling.OSM_OLD
 374         )
 375
 376         charm_list = self.build_all_charms(
 377             package_folder, skip_charm_build, is_sol004_007
 378         )
 379         return self.build_compressed_file(package_folder, charm_list, is_sol004_007)
 380
 381     def calculate_checksum(self, package_folder):
 382         """
 383         **Function to calculate the checksum given a folder**
 384
 385         :params:
 386             - package_folder: is the folder where we have the files to calculate the checksum
 387         :returns: None
 388         """
 389         self._logger.debug("")
 390         files = [
 391             f
 392             for f in glob.glob(package_folder + "/**/*.*", recursive=True)
 393             if os.path.isfile(f)
 394         ]
 395         with open("{}/checksums.txt".format(package_folder), "w+") as checksum:
 396             for file_item in files:
 397                 if "checksums.txt" in file_item:
 398                     continue
 399                 checksum.write("{}\t{}\n".format(utils.md5(file_item), file_item))
 400
 401     def create_folders(self, folders, package_type):
 402         """
 403         **Create folder given a list of folders**
 404
 405         :params:
 406             - folders: [List] list of folders paths to be created
 407             - package_type: is the type of package to be created
 408         :return: None
 409         """
 410         self._logger.debug("")
 411         for folder in folders:
 412             try:
 413                 # print("Folder {} == package_type {}".format(folder[1], package_type))
 414                 if folder[1] == package_type:
 415                     print("Creating folder:\t{}".format(folder[0]))
 416                     os.makedirs(folder[0])
 417             except FileExistsError:
 418                 pass
 419
 420     def save_file(self, file_name, file_body):
 421         """
 422         **Create a file given a name and the content**
 423
 424         :params:
 425             - file_name: is the name of the file with the relative route
 426             - file_body: is the content of the file
 427         :return: None
 428         """
 429         self._logger.debug("")
 430         print("Creating file:  \t{}".format(file_name))
 431         try:
 432             with open(file_name, "w+") as f:
 433                 f.write(file_body)
 434         except Exception as e:
 435             raise ClientException(e)
 436
 437     def generate_readme(self):
 438         """
 439         **Creates the README content**
 440
 441         :returns: readme content
 442         """
 443         self._logger.debug("")
 444         return """# Descriptor created by OSM descriptor package generated\n\n**Created on {} **""".format(
 445             time.strftime("%m/%d/%Y, %H:%M:%S", time.localtime())
 446         )
 447
 448     def generate_cloud_init(self):
 449         """
 450         **Creates the cloud-init content**
 451
 452         :returns: cloud-init content
 453         """
 454         self._logger.debug("")
 455         return "---\n#cloud-config"
 456
 457     def create_files(self, files, file_content, package_type):
 458         """
 459         **Creates the files given the file list and type**
 460
 461         :params:
 462             - files: is the list of files structure
 463             - file_content: is the content of the descriptor rendered by the template
 464             - package_type: is the type of package to filter the creation structure
 465
 466         :return: None
 467         """
 468         self._logger.debug("")
 469         for file_item, file_package, file_type in files:
 470             if package_type == file_package:
 471                 if file_type == "descriptor":
 472                     self.save_file(file_item, file_content)
 473                 elif file_type == "readme":
 474                     self.save_file(file_item, self.generate_readme())
 475                 elif file_type == "cloud_init":
 476                     self.save_file(file_item, self.generate_cloud_init())
 477
 478     def check_files_folders(self, path_list, override):
 479         """
 480         **Find files and folders missing given a directory structure {"folders": [], "files": []}**
 481
 482         :params:
 483             - path_list: is the list of files and folders to be created
 484             - override: is the flag used to indicate the creation of the list even if the file exist to override it
 485
 486         :return: Missing paths Dict
 487         """
 488         self._logger.debug("")
 489         missing_paths = {}
 490         folders = []
 491         files = []
 492         for folder in path_list.get("folders"):
 493             if not os.path.exists(folder[0]):
 494                 folders.append(folder)
 495         missing_paths["folders"] = folders
 496
 497         for file_item in path_list.get("files"):
 498             if not os.path.exists(file_item[0]) or override is True:
 499                 files.append(file_item)
 500         missing_paths["files"] = files
 501
 502         return missing_paths
 503
 504     def build_all_charms(self, package_folder, skip_charm_build, sol004_007=True):
 505         """
 506         **Read the descriptor file, check that the charms referenced are in the folder and compiles them**
 507
 508         :params:
 509             - packet_folder: is the location of the package
 510         :return: Files and Folders not found. In case of override, it will return all file list
 511         """
 512         self._logger.debug("")
 513         charms_set = set()
 514         descriptor_file = False
 515         package_type = package_handling.get_package_type(package_folder)
 516         if sol004_007 and package_type.find("TOSCA") >= 0:
 517             descriptors_paths = [
 518                 f for f in glob.glob(package_folder + "/Definitions/*.yaml")
 519             ]
 520         else:
 521             descriptors_paths = [f for f in glob.glob(package_folder + "/*.yaml")]
 522         for file in descriptors_paths:
 523             if file.endswith("nfd.yaml"):
 524                 descriptor_file = True
 525                 charms_set = self.charms_search(file, "vnf")
 526             if file.endswith("nsd.yaml"):
 527                 descriptor_file = True
 528                 charms_set = self.charms_search(file, "ns")
 529         print("List of charms in the descriptor: {}".format(charms_set))
 530         if not descriptor_file:
 531             raise ClientException(
 532                 'Descriptor filename is not correct in: {}. It should end with "nfd.yaml" or "nsd.yaml"'.format(
 533                     package_folder
 534                 )
 535             )
 536         if charms_set and not skip_charm_build:
 537             for charmName in charms_set:
 538                 if os.path.isdir(
 539                     "{}/{}charms/layers/{}".format(
 540                         package_folder, "Scripts/" if sol004_007 else "", charmName
 541                     )
 542                 ):
 543                     print(
 544                         "Building charm {}/{}charms/layers/{}".format(
 545                             package_folder, "Scripts/" if sol004_007 else "", charmName
 546                         )
 547                     )
 548                     self.charm_build(package_folder, charmName, sol004_007)
 549                     print("Charm built: {}".format(charmName))
 550                 elif os.path.isdir(
 551                     "{}/{}charms/ops/{}".format(
 552                         package_folder, "Scripts/" if sol004_007 else "", charmName
 553                     )
 554                 ):
 555                     self.charmcraft_build(package_folder, charmName)
 556                 else:
 557                     if not os.path.isdir(
 558                         "{}/{}charms/{}".format(
 559                             package_folder, "Scripts/" if sol004_007 else "", charmName
 560                         )
 561                     ) and not os.path.isfile(
 562                         "{}/{}charms/{}".format(
 563                             package_folder, "Scripts/" if sol004_007 else "", charmName
 564                         )
 565                     ):
 566                         raise ClientException(
 567                             "The charm: {} referenced in the descriptor file "
 568                             "is not present either in {}/charms or in {}/charms/layers".format(
 569                                 charmName, package_folder, package_folder
 570                             )
 571                         )
 572         self._logger.debug("Return list of charms: {}".format(charms_set))
 573         return charms_set
 574
 575     def discover_folder_structure(self, base_directory, name, override):
 576         """
 577         **Discover files and folders structure for SOL004/SOL007 descriptors given a base_directory and name**
 578
 579         :params:
 580             - base_directory: is the location of the package to be created
 581             - name: is the name of the package
 582             - override: is the flag used to indicate the creation of the list even if the file exist to override it
 583         :return: Files and Folders not found. In case of override, it will return all file list
 584         """
 585         self._logger.debug("")
 586         prefix = "{}/{}".format(base_directory, name)
 587         files_folders = {
 588             "folders": [
 589                 ("{}_ns".format(prefix), "ns"),
 590                 ("{}_ns/Licenses".format(prefix), "ns"),
 591                 ("{}_ns/Files/icons".format(prefix), "ns"),
 592                 ("{}_ns/Scripts/charms".format(prefix), "ns"),
 593                 ("{}_vnf".format(name), "vnf"),
 594                 ("{}_vnf/Licenses".format(prefix), "vnf"),
 595                 ("{}_vnf/Scripts/charms".format(prefix), "vnf"),
 596                 ("{}_vnf/Scripts/cloud_init".format(prefix), "vnf"),
 597                 ("{}_vnf/Files/images".format(prefix), "vnf"),
 598                 ("{}_vnf/Files/icons".format(prefix), "vnf"),
 599                 ("{}_vnf/Scripts/scripts".format(prefix), "vnf"),
 600                 ("{}_nst".format(prefix), "nst"),
 601                 ("{}_nst/icons".format(prefix), "nst"),
 602             ],
 603             "files": [
 604                 ("{}_ns/{}_nsd.yaml".format(prefix, name), "ns", "descriptor"),
 605                 ("{}_ns/README.md".format(prefix), "ns", "readme"),
 606                 ("{}_vnf/{}_vnfd.yaml".format(prefix, name), "vnf", "descriptor"),
 607                 (
 608                     "{}_vnf/Scripts/cloud_init/cloud-config.txt".format(prefix),
 609                     "vnf",
 610                     "cloud_init",
 611                 ),
 612                 ("{}_vnf/README.md".format(prefix), "vnf", "readme"),
 613                 ("{}_nst/{}_nst.yaml".format(prefix, name), "nst", "descriptor"),
 614                 ("{}_nst/README.md".format(prefix), "nst", "readme"),
 615             ],
 616         }
 617         missing_files_folders = self.check_files_folders(files_folders, override)
 618         # print("Missing files and folders: {}".format(missing_files_folders))
 619         return missing_files_folders
 620
 621     def charm_build(self, charms_folder, build_name, sol004_007=True):
 622         """
 623         Build the charms inside the package.
 624         params: package_folder is the name of the folder where is the charms to compile.
 625                 build_name is the name of the layer or interface
 626         """
 627         self._logger.debug("")
 628
 629         if sol004_007:
 630             os.environ["JUJU_REPOSITORY"] = "{}/Scripts/charms".format(charms_folder)
 631         else:
 632             os.environ["JUJU_REPOSITORY"] = "{}/charms".format(charms_folder)
 633
 634         os.environ["CHARM_LAYERS_DIR"] = "{}/layers".format(
 635             os.environ["JUJU_REPOSITORY"]
 636         )
 637         os.environ["CHARM_INTERFACES_DIR"] = "{}/interfaces".format(
 638             os.environ["JUJU_REPOSITORY"]
 639         )
 640
 641         if sol004_007:
 642             os.environ["CHARM_BUILD_DIR"] = "{}/Scripts/charms/builds".format(
 643                 charms_folder
 644             )
 645         else:
 646             os.environ["CHARM_BUILD_DIR"] = "{}/charms/builds".format(charms_folder)
 647
 648         if not os.path.exists(os.environ["CHARM_BUILD_DIR"]):
 649             os.makedirs(os.environ["CHARM_BUILD_DIR"])
 650         src_folder = "{}/{}".format(os.environ["CHARM_LAYERS_DIR"], build_name)
 651         result = subprocess.run(["charm", "build", "{}".format(src_folder)])
 652         if result.returncode == 1:
 653             raise ClientException("failed to build the charm: {}".format(src_folder))
 654         self._logger.verbose("charm {} built".format(src_folder))
 655
 656     def charmcraft_build(self, package_folder, charm_name):
 657         """
 658         Build the charms inside the package (new operator framework charms)
 659         params: package_folder is the name of the folder where is the charms to compile.
 660                 build_name is the name of the layer or interface
 661         """
 662         self._logger.debug("Building charm {}".format(charm_name))
 663         src_folder = f"{package_folder}/Scripts/charms/ops/{charm_name}"
 664         current_directory = os.getcwd()
 665         os.chdir(src_folder)
 666         try:
 667             result = subprocess.run(["charmcraft", "build"])
 668             if result.returncode == 1:
 669                 raise ClientException(
 670                     "failed to build the charm: {}".format(src_folder)
 671                 )
 672             subprocess.run(["rm", "-rf", f"../../{charm_name}"])
 673             subprocess.run(["mv", "build", f"../../{charm_name}"])
 674             self._logger.verbose("charm {} built".format(src_folder))
 675         finally:
 676             os.chdir(current_directory)
 677
 678     def build_compressed_file(self, package_folder, charm_list=None, sol004_007=True):
 679         if sol004_007:
 680             return self.build_zipfile(package_folder, charm_list)
 681         else:
 682             return self.build_tarfile(package_folder, charm_list)
 683
 684     def build_zipfile(self, package_folder, charm_list=None):
 685         """
 686         Creates a zip file given a package_folder
 687         params: package_folder is the name of the folder to be packaged
 688         returns: .zip name
 689         """
 690         self._logger.debug("")
 691         cwd = None
 692         try:
 693             directory_name, package_name = self.create_temp_dir_sol004_007(
 694                 package_folder, charm_list
 695             )
 696             cwd = os.getcwd()
 697             os.chdir(directory_name)
 698             package_type = package_handling.get_package_type(package_folder)
 699             print(package_type)
 700
 701             if (
 702                 package_handling.SOL007 == package_type
 703                 or package_handling.SOL007_TOSCA == package_type
 704             ):
 705                 the_package = SOL007Package(package_folder)
 706             elif (
 707                 package_handling.SOL004 == package_type
 708                 or package_handling.SOL004_TOSCA == package_type
 709             ):
 710                 the_package = SOL004Package(package_folder)
 711
 712             the_package.create_or_update_metadata_file()
 713
 714             the_zip_package = shutil.make_archive(
 715                 os.path.join(cwd, package_name),
 716                 "zip",
 717                 os.path.join(directory_name, package_name),
 718             )
 719
 720             print("Package created: {}".format(the_zip_package))
 721
 722             return the_zip_package
 723
 724         except Exception as exc:
 725             raise ClientException(
 726                 "failure during build of zip file (create temp dir, calculate checksum, "
 727                 "zip file): {}".format(exc)
 728             )
 729         finally:
 730             if cwd:
 731                 os.chdir(cwd)
 732             shutil.rmtree(os.path.join(package_folder, "tmp"))
 733
 734     def build_tarfile(self, package_folder, charm_list=None):
 735         """
 736         Creates a .tar.gz file given a package_folder
 737         params: package_folder is the name of the folder to be packaged
 738         returns: .tar.gz name
 739         """
 740         self._logger.debug("")
 741         cwd = None
 742         try:
 743             directory_name, package_name = self.create_temp_dir(
 744                 package_folder, charm_list
 745             )
 746             cwd = os.getcwd()
 747             os.chdir(directory_name)
 748             self.calculate_checksum(package_name)
 749             with tarfile.open("{}.tar.gz".format(package_name), mode="w:gz") as archive:
 750                 print("Adding File: {}".format(package_name))
 751                 archive.add("{}".format(package_name), recursive=True)
 752             # return "Created {}.tar.gz".format(package_folder)
 753             # self.build("{}".format(os.path.basename(package_folder)))
 754             os.chdir(cwd)
 755             cwd = None
 756             created_package = "{}/{}.tar.gz".format(
 757                 os.path.dirname(package_folder) or ".", package_name
 758             )
 759             os.rename(
 760                 "{}/{}.tar.gz".format(directory_name, package_name), created_package
 761             )
 762             os.rename(
 763                 "{}/{}/checksums.txt".format(directory_name, package_name),
 764                 "{}/checksums.txt".format(package_folder),
 765             )
 766             print("Package created: {}".format(created_package))
 767             return created_package
 768         except Exception as exc:
 769             raise ClientException(
 770                 "failure during build of targz file (create temp dir, calculate checksum, "
 771                 "tar.gz file): {}".format(exc)
 772             )
 773         finally:
 774             if cwd:
 775                 os.chdir(cwd)
 776             shutil.rmtree(os.path.join(package_folder, "tmp"))
 777
 778     def create_temp_dir(self, package_folder, charm_list=None):
 779         """
 780         Method to create a temporary folder where we can move the files in package_folder
 781         """
 782         self._logger.debug("")
 783         ignore_patterns = ".gitignore"
 784         ignore = shutil.ignore_patterns(ignore_patterns)
 785         directory_name = os.path.abspath(package_folder)
 786         package_name = os.path.basename(directory_name)
 787         directory_name += "/tmp"
 788         os.makedirs("{}/{}".format(directory_name, package_name), exist_ok=True)
 789         self._logger.debug("Makedirs DONE: {}/{}".format(directory_name, package_name))
 790         for item in os.listdir(package_folder):
 791             self._logger.debug("Item: {}".format(item))
 792             if item != "tmp":
 793                 s = os.path.join(package_folder, item)
 794                 d = os.path.join(os.path.join(directory_name, package_name), item)
 795                 if os.path.isdir(s):
 796                     if item == "charms":
 797                         os.makedirs(d, exist_ok=True)
 798                         s_builds = os.path.join(s, "builds")
 799                         for charm in charm_list:
 800                             self._logger.debug("Copying charm {}".format(charm))
 801                             if charm in os.listdir(s):
 802                                 s_charm = os.path.join(s, charm)
 803                             elif charm in os.listdir(s_builds):
 804                                 s_charm = os.path.join(s_builds, charm)
 805                             else:
 806                                 raise ClientException(
 807                                     "The charm {} referenced in the descriptor file "
 808                                     "could not be found in {}/charms or in {}/charms/builds".format(
 809                                         charm, package_folder, package_folder
 810                                     )
 811                                 )
 812                             d_temp = os.path.join(d, charm)
 813                             self._logger.debug(
 814                                 "Copying tree: {} -> {}".format(s_charm, d_temp)
 815                             )
 816                             if os.path.isdir(s_charm):
 817                                 shutil.copytree(
 818                                     s_charm, d_temp, symlinks=True, ignore=ignore
 819                                 )
 820                             else:
 821                                 shutil.copy2(s_charm, d_temp)
 822                             self._logger.debug("DONE")
 823                     else:
 824                         self._logger.debug("Copying tree: {} -> {}".format(s, d))
 825                         shutil.copytree(s, d, symlinks=True, ignore=ignore)
 826                         self._logger.debug("DONE")
 827                 else:
 828                     if item in ignore_patterns:
 829                         continue
 830                     self._logger.debug("Copying file: {} -> {}".format(s, d))
 831                     shutil.copy2(s, d)
 832                     self._logger.debug("DONE")
 833         return directory_name, package_name
 834
 835     def copy_tree(self, s, d, ignore):
 836         self._logger.debug("Copying tree: {} -> {}".format(s, d))
 837         shutil.copytree(s, d, symlinks=True, ignore=ignore)
 838         self._logger.debug("DONE")
 839
 840     def create_temp_dir_sol004_007(self, package_folder, charm_list=None):
 841         """
 842         Method to create a temporary folder where we can move the files in package_folder
 843         """
 844         self._logger.debug("")
 845         ignore_patterns = ".gitignore"
 846         ignore = shutil.ignore_patterns(ignore_patterns)
 847         directory_name = os.path.abspath(package_folder)
 848         package_name = os.path.basename(directory_name)
 849         directory_name += "/tmp"
 850         os.makedirs("{}/{}".format(directory_name, package_name), exist_ok=True)
 851         self._logger.debug("Makedirs DONE: {}/{}".format(directory_name, package_name))
 852         for item in os.listdir(package_folder):
 853             self._logger.debug("Item: {}".format(item))
 854             if item != "tmp":
 855                 s = os.path.join(package_folder, item)
 856                 d = os.path.join(os.path.join(directory_name, package_name), item)
 857                 if os.path.isdir(s):
 858                     if item == "Scripts":
 859                         os.makedirs(d, exist_ok=True)
 860                         scripts_folder = s
 861                         for script_item in os.listdir(scripts_folder):
 862                             scripts_destination_folder = os.path.join(d, script_item)
 863                             if script_item == "charms":
 864                                 s_builds = os.path.join(
 865                                     scripts_folder, script_item, "builds"
 866                                 )
 867                                 for charm in charm_list:
 868                                     self._logger.debug("Copying charm {}".format(charm))
 869                                     if charm in os.listdir(
 870                                         os.path.join(scripts_folder, script_item)
 871                                     ):
 872                                         s_charm = os.path.join(
 873                                             scripts_folder, script_item, charm
 874                                         )
 875                                     elif charm in os.listdir(s_builds):
 876                                         s_charm = os.path.join(s_builds, charm)
 877                                     else:
 878                                         raise ClientException(
 879                                             "The charm {} referenced in the descriptor file "
 880                                             "could not be found in {}/charms or in {}/charms/builds".format(
 881                                                 charm, package_folder, package_folder
 882                                             )
 883                                         )
 884                                     d_temp = os.path.join(
 885                                         scripts_destination_folder, charm
 886                                     )
 887                                     self.copy_tree(s_charm, d_temp, ignore)
 888                             else:
 889                                 self.copy_tree(
 890                                     os.path.join(scripts_folder, script_item),
 891                                     scripts_destination_folder,
 892                                     ignore,
 893                                 )
 894                     else:
 895                         self.copy_tree(s, d, ignore)
 896                 else:
 897                     if item in ignore_patterns:
 898                         continue
 899                     self._logger.debug("Copying file: {} -> {}".format(s, d))
 900                     shutil.copy2(s, d)
 901                     self._logger.debug("DONE")
 902         return directory_name, package_name
 903
 904     def charms_search(self, descriptor_file, desc_type):
 905         self._logger.debug(
 906             "descriptor_file: {}, desc_type: {}".format(descriptor_file, desc_type)
 907         )
 908         charms_set = set()
 909         with open("{}".format(descriptor_file)) as yaml_desc:
 910             descriptor_dict = yaml.safe_load(yaml_desc)
 911             # self._logger.debug("\n"+yaml.safe_dump(descriptor_dict, indent=4, default_flow_style=False))
 912
 913             if (
 914                 desc_type == "vnf"
 915                 and (
 916                     "vnfd:vnfd-catalog" in descriptor_dict
 917                     or "vnfd-catalog" in descriptor_dict
 918                 )
 919             ) or (
 920                 desc_type == "ns"
 921                 and (
 922                     "nsd:nsd-catalog" in descriptor_dict
 923                     or "nsd-catalog" in descriptor_dict
 924                 )
 925             ):
 926                 charms_set = self._charms_search_on_osm_im_dict(
 927                     descriptor_dict, desc_type
 928                 )
 929             else:
 930                 if desc_type == "ns":
 931                     get_charm_list = self._charms_search_on_nsd_sol006_dict
 932                 elif desc_type == "vnf":
 933                     get_charm_list = self._charms_search_on_vnfd_sol006_dict
 934                 else:
 935                     raise Exception("Bad descriptor type")
 936                 charms_set = get_charm_list(descriptor_dict)
 937         return charms_set
 938
 939     def _charms_search_on_osm_im_dict(self, osm_im_dict, desc_type):
 940         self._logger.debug("")
 941         charms_set = set()
 942         for k1, v1 in osm_im_dict.items():
 943             for k2, v2 in v1.items():
 944                 for entry in v2:
 945                     if "{}-configuration".format(desc_type) in entry:
 946                         vnf_config = entry["{}-configuration".format(desc_type)]
 947                         for k3, v3 in vnf_config.items():
 948                             if "charm" in v3:
 949                                 charms_set.add((v3["charm"]))
 950                     if "vdu" in entry:
 951                         vdus = entry["vdu"]
 952                         for vdu in vdus:
 953                             if "vdu-configuration" in vdu:
 954                                 for k4, v4 in vdu["vdu-configuration"].items():
 955                                     if "charm" in v4:
 956                                         charms_set.add((v4["charm"]))
 957         return charms_set
 958
 959     def _charms_search_on_vnfd_sol006_dict(self, sol006_dict):
 960         self._logger.debug("")
 961         charms_set = set()
 962         dfs = sol006_dict.get("vnfd", {}).get("df", [])
 963         for df in dfs:
 964             day_1_2s = (
 965                 df.get("lcm-operations-configuration", {})
 966                 .get("operate-vnf-op-config", {})
 967                 .get("day1-2")
 968             )
 969             if day_1_2s is not None:
 970                 for day_1_2 in day_1_2s:
 971                     exec_env_list = day_1_2.get("execution-environment-list", [])
 972                     for exec_env in exec_env_list:
 973                         if "juju" in exec_env and "charm" in exec_env["juju"]:
 974                             charms_set.add(exec_env["juju"]["charm"])
 975         return charms_set
 976
 977     def _charms_search_on_nsd_sol006_dict(self, sol006_dict):
 978         self._logger.debug("")
 979         charms_set = set()
 980         nsd_list = sol006_dict.get("nsd", {}).get("nsd", [])
 981         for nsd in nsd_list:
 982             charm = nsd.get("ns-configuration", {}).get("juju", {}).get("charm")
 983             if charm:
 984                 charms_set.add(charm)
 985         return charms_set