osmclient/common/package_tool.py

   1 # /bin/env python3
   2 # Copyright 2019 ATOS
   3 #
   4 # All Rights Reserved.
   5 #
   6 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 #    not use this file except in compliance with the License. You may obtain
   8 #    a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #    Unless required by applicable law or agreed to in writing, software
  13 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 #    License for the specific language governing permissions and limitations
  16 #    under the License.
  17
  18 import glob
  19 import logging
  20 import os
  21 import shutil
  22 import subprocess
  23 import tarfile
  24 import time
  25 from jinja2 import Environment, PackageLoader
  26 from osm_im.validation import Validation as validation_im
  27 from osm_im.validation import ValidationException
  28 from osm_im import im_translation
  29 from osmclient.common import package_handling as package_handling
  30 from osmclient.common.exceptions import ClientException
  31 from osmclient.common import utils
  32 from .sol004_package import SOL004Package
  33 from .sol007_package import SOL007Package
  34 import yaml
  35
  36
  37 class PackageTool(object):
  38     def __init__(self, client=None):
  39         self._client = client
  40         self._logger = logging.getLogger("osmclient")
  41         self._validator = validation_im()
  42
  43     def create(
  44         self,
  45         package_type,
  46         base_directory,
  47         package_name,
  48         override,
  49         image,
  50         vdus,
  51         vcpu,
  52         memory,
  53         storage,
  54         interfaces,
  55         vendor,
  56         detailed,
  57         netslice_subnets,
  58         netslice_vlds,
  59         old,
  60     ):
  61         """
  62         **Create a package descriptor**
  63
  64         :params:
  65             - package_type: [vnf, ns, nst]
  66             - base directory: path of destination folder
  67             - package_name: is the name of the package to be created
  68             - image: specify the image of the vdu
  69             - vcpu: number of virtual cpus of the vdu
  70             - memory: amount of memory in MB pf the vdu
  71             - storage: amount of storage in GB of the vdu
  72             - interfaces: number of interfaces besides management interface
  73             - vendor: vendor name of the vnf/ns
  74             - detailed: include all possible values for NSD, VNFD, NST
  75             - netslice_subnets: number of netslice_subnets for the NST
  76             - netslice_vlds: number of virtual link descriptors for the NST
  77             - old: flag to create a descriptor using the previous OSM format (pre SOL006, OSM<9)
  78
  79         :return: status
  80         """
  81         self._logger.debug("")
  82         # print("location: {}".format(osmclient.__path__))
  83         file_loader = PackageLoader("osmclient")
  84         env = Environment(loader=file_loader, autoescape=True)
  85         if package_type == "ns":
  86             template = env.get_template("nsd.yaml.j2" if not old else "nsd_old.yaml.j2")
  87             content = {
  88                 "name": package_name,
  89                 "vendor": vendor,
  90                 "vdus": vdus,
  91                 "clean": False,
  92                 "interfaces": interfaces,
  93                 "detailed": detailed,
  94             }
  95         elif package_type == "vnf":
  96             template = env.get_template(
  97                 "vnfd.yaml.j2" if not old else "vnfd_old.yaml.j2"
  98             )
  99             content = {
 100                 "name": package_name,
 101                 "vendor": vendor,
 102                 "vdus": vdus,
 103                 "clean": False,
 104                 "interfaces": interfaces,
 105                 "image": image,
 106                 "vcpu": vcpu,
 107                 "memory": memory,
 108                 "storage": storage,
 109                 "detailed": detailed,
 110             }
 111         elif package_type == "nst":
 112             # TODO: repo-index did not support nst in OSM<9, no changes in template
 113             template = env.get_template("nst.yaml.j2")
 114             content = {
 115                 "name": package_name,
 116                 "vendor": vendor,
 117                 "interfaces": interfaces,
 118                 "netslice_subnets": netslice_subnets,
 119                 "netslice_vlds": netslice_vlds,
 120                 "detailed": detailed,
 121             }
 122         else:
 123             raise ClientException(
 124                 "Wrong descriptor type {}. Options: ns, vnf, nst".format(package_type)
 125             )
 126
 127         self._logger.debug("To be rendered: {}".format(content))
 128         output = template.render(content)
 129         self._logger.debug(output)
 130
 131         structure = self.discover_folder_structure(
 132             base_directory, package_name, override
 133         )
 134         if structure.get("folders"):
 135             self.create_folders(structure["folders"], package_type)
 136         if structure.get("files"):
 137             self.create_files(structure["files"], output, package_type)
 138         return "Created"
 139
 140     def validate(self, base_directory, recursive=True, old_format=False):
 141         """
 142         **Validate OSM Descriptors given a path**
 143
 144         :params:
 145             - base_directory is the root path for all descriptors
 146
 147         :return: List of dict of validated descriptors. keys: type, path, valid, error
 148         """
 149         self._logger.debug("")
 150         table = []
 151         if recursive:
 152             descriptors_paths = [
 153                 f for f in glob.glob(base_directory + "/**/*.yaml", recursive=recursive)
 154             ]
 155         else:
 156             descriptors_paths = [
 157                 f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)
 158             ]
 159         self._logger.info("Base directory: {}".format(base_directory))
 160         self._logger.info(
 161             "{} Descriptors found to validate".format(len(descriptors_paths))
 162         )
 163         for desc_path in descriptors_paths:
 164             with open(desc_path) as descriptor_file:
 165                 descriptor_data = descriptor_file.read()
 166             desc_type = "-"
 167             try:
 168                 # TODO: refactor validation_im.yaml_validation to @staticmethod
 169                 desc_type, descriptor_data = validation_im.yaml_validation(
 170                     self, descriptor_data
 171                 )
 172                 self._logger.debug(f"Validate {desc_type} {descriptor_data}")
 173                 if not old_format:
 174                     if desc_type == "vnfd" or desc_type == "nsd":
 175                         self._logger.error(
 176                             "OSM descriptor '{}' written in an unsupported format. Please update to ETSI SOL006 format".format(
 177                                 desc_path
 178                             )
 179                         )
 180                         self._logger.warning(
 181                             "Package validation skipped. It can still be done with 'osm package-validate --old'"
 182                         )
 183                         self._logger.warning(
 184                             "Package build can still be done with 'osm package-build --skip-validation'"
 185                         )
 186                         raise Exception("Not SOL006 format")
 187                 validation_im.pyangbind_validation(self, desc_type, descriptor_data)
 188                 table.append(
 189                     {"type": desc_type, "path": desc_path, "valid": "OK", "error": "-"}
 190                 )
 191             except Exception as e:
 192                 self._logger.error(f"Validation error: {e}", exc_info=True)
 193                 table.append(
 194                     {
 195                         "type": desc_type,
 196                         "path": desc_path,
 197                         "valid": "ERROR",
 198                         "error": str(e),
 199                     }
 200                 )
 201             self._logger.debug(table[-1])
 202         return table
 203
 204     def translate(self, base_directory, recursive=True, dryrun=False):
 205         """
 206         **Translate OSM Packages given a path**
 207
 208         :params:
 209             - base_directory is the root path for all packages
 210
 211         :return: List of dict of translated packages. keys: current type, new type, path, valid, translated, error
 212         """
 213         self._logger.debug("")
 214         table = []
 215         if recursive:
 216             descriptors_paths = [
 217                 f for f in glob.glob(base_directory + "/**/*.yaml", recursive=recursive)
 218             ]
 219         else:
 220             descriptors_paths = [
 221                 f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)
 222             ]
 223         print("Base directory: {}".format(base_directory))
 224         print("{} Descriptors found to validate".format(len(descriptors_paths)))
 225         for desc_path in descriptors_paths:
 226             with open(desc_path) as descriptor_file:
 227                 descriptor_data = descriptor_file.read()
 228             desc_type = "-"
 229             try:
 230                 desc_type, descriptor_data = validation_im.yaml_validation(
 231                     self, descriptor_data
 232                 )
 233                 self._logger.debug("desc_type: {}".format(desc_type))
 234                 self._logger.debug("descriptor_data:\n{}".format(descriptor_data))
 235                 self._validator.pyangbind_validation(desc_type, descriptor_data)
 236                 if not (desc_type == "vnfd" or desc_type == "nsd"):
 237                     table.append(
 238                         {
 239                             "current type": desc_type,
 240                             "new type": desc_type,
 241                             "path": desc_path,
 242                             "valid": "OK",
 243                             "translated": "N/A",
 244                             "error": "-",
 245                         }
 246                     )
 247                 else:
 248                     new_desc_type = desc_type
 249                     try:
 250                         sol006_model = yaml.safe_dump(
 251                             im_translation.translate_im_model_to_sol006(
 252                                 descriptor_data
 253                             ),
 254                             indent=4,
 255                             default_flow_style=False,
 256                         )
 257                         (
 258                             new_desc_type,
 259                             new_descriptor_data,
 260                         ) = self._validator.yaml_validation(sol006_model)
 261                         self._validator.pyangbind_validation(
 262                             new_desc_type, new_descriptor_data
 263                         )
 264                         if not dryrun:
 265                             with open(desc_path, "w") as descriptor_file:
 266                                 descriptor_file.write(sol006_model)
 267                         table.append(
 268                             {
 269                                 "current type": desc_type,
 270                                 "new type": new_desc_type,
 271                                 "path": desc_path,
 272                                 "valid": "OK",
 273                                 "translated": "OK",
 274                                 "error": "-",
 275                             }
 276                         )
 277                     except ValidationException as ve2:
 278                         table.append(
 279                             {
 280                                 "current type": desc_type,
 281                                 "new type": new_desc_type,
 282                                 "path": desc_path,
 283                                 "valid": "OK",
 284                                 "translated": "ERROR",
 285                                 "error": "Error in the post-validation: {}".format(
 286                                     str(ve2)
 287                                 ),
 288                             }
 289                         )
 290                     except Exception as e2:
 291                         table.append(
 292                             {
 293                                 "current type": desc_type,
 294                                 "new type": new_desc_type,
 295                                 "path": desc_path,
 296                                 "valid": "OK",
 297                                 "translated": "ERROR",
 298                                 "error": "Error in the translation: {}".format(str(e2)),
 299                             }
 300                         )
 301             except ValidationException as ve:
 302                 table.append(
 303                     {
 304                         "current type": desc_type,
 305                         "new type": "N/A",
 306                         "path": desc_path,
 307                         "valid": "ERROR",
 308                         "translated": "N/A",
 309                         "error": "Error in the pre-validation: {}".format(str(ve)),
 310                     }
 311                 )
 312             except Exception as e:
 313                 table.append(
 314                     {
 315                         "current type": desc_type,
 316                         "new type": "N/A",
 317                         "path": desc_path,
 318                         "valid": "ERROR",
 319                         "translated": "N/A",
 320                         "error": str(e),
 321                     }
 322                 )
 323         return table
 324
 325     def descriptor_translate(self, descriptor_file):
 326         """
 327         **Translate input descriptor file from Rel EIGHT OSM to SOL006**
 328
 329         :params:
 330             - base_directory is the root path for all packages
 331
 332         :return: YAML descriptor in the new format
 333         """
 334         self._logger.debug("")
 335         with open(descriptor_file, "r") as df:
 336             im_model = yaml.safe_load(df.read())
 337         sol006_model = im_translation.translate_im_model_to_sol006(im_model)
 338         return yaml.safe_dump(sol006_model, indent=4, default_flow_style=False)
 339
 340     def build(self, package_folder, skip_validation=False, skip_charm_build=False):
 341         """
 342         **Creates a .tar.gz file given a package_folder**
 343
 344         :params:
 345             - package_folder: is the name of the folder to be packaged
 346             - skip_validation: is the flag to validate or not the descriptors on the folder before build
 347
 348         :returns: message result for the build process
 349         """
 350         self._logger.debug("")
 351         package_folder = package_folder.rstrip("/")
 352         if not os.path.exists("{}".format(package_folder)):
 353             return "Fail, package is not in the specified path"
 354         if not skip_validation:
 355             print("Validating package {}".format(package_folder))
 356             results = self.validate(package_folder, recursive=False)
 357             if results:
 358                 for result in results:
 359                     if result["valid"] != "OK":
 360                         raise ClientException(
 361                             "There was an error validating the file {} with error: {}".format(
 362                                 result["path"], result["error"]
 363                             )
 364                         )
 365                 print("Validation OK")
 366             else:
 367                 raise ClientException(
 368                     "No descriptor file found in: {}".format(package_folder)
 369                 )
 370
 371         is_sol004_007 = (
 372             package_handling.get_package_type(package_folder)
 373             != package_handling.OSM_OLD
 374         )
 375
 376         charm_list = self.build_all_charms(
 377             package_folder, skip_charm_build, is_sol004_007
 378         )
 379         return self.build_compressed_file(package_folder, charm_list, is_sol004_007)
 380
 381     def calculate_checksum(self, package_folder):
 382         """
 383         **Function to calculate the checksum given a folder**
 384
 385         :params:
 386             - package_folder: is the folder where we have the files to calculate the checksum
 387         :returns: None
 388         """
 389         self._logger.debug("")
 390         files = [
 391             f
 392             for f in glob.glob(package_folder + "/**/*.*", recursive=True)
 393             if os.path.isfile(f)
 394         ]
 395         with open("{}/checksums.txt".format(package_folder), "w+") as checksum:
 396             for file_item in files:
 397                 if "checksums.txt" in file_item:
 398                     continue
 399                 checksum.write("{}\t{}\n".format(utils.md5(file_item), file_item))
 400
 401     def create_folders(self, folders, package_type):
 402         """
 403         **Create folder given a list of folders**
 404
 405         :params:
 406             - folders: [List] list of folders paths to be created
 407             - package_type: is the type of package to be created
 408         :return: None
 409         """
 410         self._logger.debug("")
 411         for folder in folders:
 412             try:
 413                 # print("Folder {} == package_type {}".format(folder[1], package_type))
 414                 if folder[1] == package_type:
 415                     print("Creating folder:\t{}".format(folder[0]))
 416                     os.makedirs(folder[0])
 417             except FileExistsError:
 418                 pass
 419
 420     def save_file(self, file_name, file_body):
 421         """
 422         **Create a file given a name and the content**
 423
 424         :params:
 425             - file_name: is the name of the file with the relative route
 426             - file_body: is the content of the file
 427         :return: None
 428         """
 429         self._logger.debug("")
 430         print("Creating file:  \t{}".format(file_name))
 431         try:
 432             with open(file_name, "w+") as f:
 433                 f.write(file_body)
 434         except Exception as e:
 435             raise ClientException(e)
 436
 437     def generate_readme(self):
 438         """
 439         **Creates the README content**
 440
 441         :returns: readme content
 442         """
 443         self._logger.debug("")
 444         return """# Descriptor created by OSM descriptor package generated\n\n**Created on {} **""".format(
 445             time.strftime("%m/%d/%Y, %H:%M:%S", time.localtime())
 446         )
 447
 448     def generate_cloud_init(self):
 449         """
 450         **Creates the cloud-init content**
 451
 452         :returns: cloud-init content
 453         """
 454         self._logger.debug("")
 455         return "---\n#cloud-config"
 456
 457     def create_files(self, files, file_content, package_type):
 458         """
 459         **Creates the files given the file list and type**
 460
 461         :params:
 462             - files: is the list of files structure
 463             - file_content: is the content of the descriptor rendered by the template
 464             - package_type: is the type of package to filter the creation structure
 465
 466         :return: None
 467         """
 468         self._logger.debug("")
 469         for file_item, file_package, file_type in files:
 470             if package_type == file_package:
 471                 if file_type == "descriptor":
 472                     self.save_file(file_item, file_content)
 473                 elif file_type == "readme":
 474                     self.save_file(file_item, self.generate_readme())
 475                 elif file_type == "cloud_init":
 476                     self.save_file(file_item, self.generate_cloud_init())
 477
 478     def check_files_folders(self, path_list, override):
 479         """
 480         **Find files and folders missing given a directory structure {"folders": [], "files": []}**
 481
 482         :params:
 483             - path_list: is the list of files and folders to be created
 484             - override: is the flag used to indicate the creation of the list even if the file exist to override it
 485
 486         :return: Missing paths Dict
 487         """
 488         self._logger.debug("")
 489         missing_paths = {}
 490         folders = []
 491         files = []
 492         for folder in path_list.get("folders"):
 493             if not os.path.exists(folder[0]):
 494                 folders.append(folder)
 495         missing_paths["folders"] = folders
 496
 497         for file_item in path_list.get("files"):
 498             if not os.path.exists(file_item[0]) or override is True:
 499                 files.append(file_item)
 500         missing_paths["files"] = files
 501
 502         return missing_paths
 503
 504     def build_all_charms(self, package_folder, skip_charm_build, sol004_007=True):
 505         """
 506         **Read the descriptor file, check that the charms referenced are in the folder and compiles them**
 507
 508         :params:
 509             - packet_folder: is the location of the package
 510         :return: Files and Folders not found. In case of override, it will return all file list
 511         """
 512         self._logger.debug("")
 513         charms_set = set()
 514         descriptor_file = False
 515         package_type = package_handling.get_package_type(package_folder)
 516         if sol004_007 and package_type.find("TOSCA") >= 0:
 517             descriptors_paths = [
 518                 f for f in glob.glob(package_folder + "/Definitions/*.yaml")
 519             ]
 520         else:
 521             descriptors_paths = [f for f in glob.glob(package_folder + "/*.yaml")]
 522         if len(descriptors_paths) == 1:
 523             # The base folder usually has a single yaml file with the descriptor
 524             descriptor_file = True
 525             pkg_type = utils.get_key_val_from_descriptor(descriptors_paths[0])
 526             if pkg_type is None:
 527                 raise ClientException("Cannot determine package type")
 528             if pkg_type["type"] == "nsd":
 529                 charms_set = self.charms_search(descriptors_paths[0], "ns")
 530             else:
 531                 charms_set = self.charms_search(descriptors_paths[0], "vnf")
 532         elif len(descriptors_paths) > 1:
 533             for file in descriptors_paths:
 534                 if file.endswith("nfd.yaml"):
 535                     descriptor_file = True
 536                     charms_set = self.charms_search(file, "vnf")
 537                 if file.endswith("nsd.yaml"):
 538                     descriptor_file = True
 539                     charms_set = self.charms_search(file, "ns")
 540         else:
 541             raise ClientException(
 542                 "Package folder does not contain valid descriptor files '*.yaml'"
 543             )
 544         print("List of charms in the descriptor: {}".format(charms_set))
 545         if not descriptor_file:
 546             raise ClientException(
 547                 'Descriptor filename is not correct in: {}. It should end with "nfd.yaml" or "nsd.yaml"'.format(
 548                     package_folder
 549                 )
 550             )
 551         if charms_set and not skip_charm_build:
 552             for charmName in charms_set:
 553                 if os.path.isdir(
 554                     "{}/{}charms/layers/{}".format(
 555                         package_folder, "Scripts/" if sol004_007 else "", charmName
 556                     )
 557                 ):
 558                     print(
 559                         "Building charm {}/{}charms/layers/{}".format(
 560                             package_folder, "Scripts/" if sol004_007 else "", charmName
 561                         )
 562                     )
 563                     self.charm_build(package_folder, charmName, sol004_007)
 564                     print("Charm built: {}".format(charmName))
 565                 elif os.path.isdir(
 566                     "{}/{}charms/ops/{}".format(
 567                         package_folder, "Scripts/" if sol004_007 else "", charmName
 568                     )
 569                 ):
 570                     self.charmcraft_build(package_folder, charmName)
 571                 else:
 572                     if not os.path.isdir(
 573                         "{}/{}charms/{}".format(
 574                             package_folder, "Scripts/" if sol004_007 else "", charmName
 575                         )
 576                     ) and not os.path.isfile(
 577                         "{}/{}charms/{}".format(
 578                             package_folder, "Scripts/" if sol004_007 else "", charmName
 579                         )
 580                     ):
 581                         raise ClientException(
 582                             "The charm: {} referenced in the descriptor file "
 583                             "is not present either in {}/charms or in {}/charms/layers".format(
 584                                 charmName, package_folder, package_folder
 585                             )
 586                         )
 587         self._logger.debug("Return list of charms: {}".format(charms_set))
 588         return charms_set
 589
 590     def discover_folder_structure(self, base_directory, name, override):
 591         """
 592         **Discover files and folders structure for SOL004/SOL007 descriptors given a base_directory and name**
 593
 594         :params:
 595             - base_directory: is the location of the package to be created
 596             - name: is the name of the package
 597             - override: is the flag used to indicate the creation of the list even if the file exist to override it
 598         :return: Files and Folders not found. In case of override, it will return all file list
 599         """
 600         self._logger.debug("")
 601         prefix = "{}/{}".format(base_directory, name)
 602         files_folders = {
 603             "folders": [
 604                 ("{}_ns".format(prefix), "ns"),
 605                 ("{}_ns/Licenses".format(prefix), "ns"),
 606                 ("{}_ns/Files/icons".format(prefix), "ns"),
 607                 ("{}_ns/Scripts/charms".format(prefix), "ns"),
 608                 ("{}_vnf".format(name), "vnf"),
 609                 ("{}_vnf/Licenses".format(prefix), "vnf"),
 610                 ("{}_vnf/Scripts/charms".format(prefix), "vnf"),
 611                 ("{}_vnf/Scripts/cloud_init".format(prefix), "vnf"),
 612                 ("{}_vnf/Files/images".format(prefix), "vnf"),
 613                 ("{}_vnf/Files/icons".format(prefix), "vnf"),
 614                 ("{}_vnf/Scripts/scripts".format(prefix), "vnf"),
 615                 ("{}_nst".format(prefix), "nst"),
 616                 ("{}_nst/icons".format(prefix), "nst"),
 617             ],
 618             "files": [
 619                 ("{}_ns/{}_nsd.yaml".format(prefix, name), "ns", "descriptor"),
 620                 ("{}_ns/README.md".format(prefix), "ns", "readme"),
 621                 ("{}_vnf/{}_vnfd.yaml".format(prefix, name), "vnf", "descriptor"),
 622                 (
 623                     "{}_vnf/Scripts/cloud_init/cloud-config.txt".format(prefix),
 624                     "vnf",
 625                     "cloud_init",
 626                 ),
 627                 ("{}_vnf/README.md".format(prefix), "vnf", "readme"),
 628                 ("{}_nst/{}_nst.yaml".format(prefix, name), "nst", "descriptor"),
 629                 ("{}_nst/README.md".format(prefix), "nst", "readme"),
 630             ],
 631         }
 632         missing_files_folders = self.check_files_folders(files_folders, override)
 633         # print("Missing files and folders: {}".format(missing_files_folders))
 634         return missing_files_folders
 635
 636     def charm_build(self, charms_folder, build_name, sol004_007=True):
 637         """
 638         Build the charms inside the package.
 639         params: package_folder is the name of the folder where is the charms to compile.
 640                 build_name is the name of the layer or interface
 641         """
 642         self._logger.debug("")
 643
 644         if sol004_007:
 645             os.environ["JUJU_REPOSITORY"] = "{}/Scripts/charms".format(charms_folder)
 646         else:
 647             os.environ["JUJU_REPOSITORY"] = "{}/charms".format(charms_folder)
 648
 649         os.environ["CHARM_LAYERS_DIR"] = "{}/layers".format(
 650             os.environ["JUJU_REPOSITORY"]
 651         )
 652         os.environ["CHARM_INTERFACES_DIR"] = "{}/interfaces".format(
 653             os.environ["JUJU_REPOSITORY"]
 654         )
 655
 656         if sol004_007:
 657             os.environ["CHARM_BUILD_DIR"] = "{}/Scripts/charms/builds".format(
 658                 charms_folder
 659             )
 660         else:
 661             os.environ["CHARM_BUILD_DIR"] = "{}/charms/builds".format(charms_folder)
 662
 663         if not os.path.exists(os.environ["CHARM_BUILD_DIR"]):
 664             os.makedirs(os.environ["CHARM_BUILD_DIR"])
 665         src_folder = "{}/{}".format(os.environ["CHARM_LAYERS_DIR"], build_name)
 666         result = subprocess.run(["charm", "build", "{}".format(src_folder)])
 667         if result.returncode == 1:
 668             raise ClientException("failed to build the charm: {}".format(src_folder))
 669         self._logger.verbose("charm {} built".format(src_folder))
 670
 671     def charmcraft_build(self, package_folder, charm_name):
 672         """
 673         Build the charms inside the package (new operator framework charms)
 674         params: package_folder is the name of the folder where is the charms to compile.
 675                 build_name is the name of the layer or interface
 676         """
 677         self._logger.debug("Building charm {}".format(charm_name))
 678         src_folder = f"{package_folder}/Scripts/charms/ops/{charm_name}"
 679         current_directory = os.getcwd()
 680         os.chdir(src_folder)
 681         try:
 682             result = subprocess.run(["charmcraft", "build"])
 683             if result.returncode == 1:
 684                 raise ClientException(
 685                     "failed to build the charm: {}".format(src_folder)
 686                 )
 687             subprocess.run(["rm", "-rf", f"../../{charm_name}"])
 688             subprocess.run(["mv", "build", f"../../{charm_name}"])
 689             self._logger.verbose("charm {} built".format(src_folder))
 690         finally:
 691             os.chdir(current_directory)
 692
 693     def build_compressed_file(self, package_folder, charm_list=None, sol004_007=True):
 694         if sol004_007:
 695             return self.build_zipfile(package_folder, charm_list)
 696         else:
 697             return self.build_tarfile(package_folder, charm_list)
 698
 699     def build_zipfile(self, package_folder, charm_list=None):
 700         """
 701         Creates a zip file given a package_folder
 702         params: package_folder is the name of the folder to be packaged
 703         returns: .zip name
 704         """
 705         self._logger.debug("")
 706         cwd = None
 707         try:
 708             directory_name, package_name = self.create_temp_dir_sol004_007(
 709                 package_folder, charm_list
 710             )
 711             cwd = os.getcwd()
 712             os.chdir(directory_name)
 713             package_type = package_handling.get_package_type(package_folder)
 714             print(package_type)
 715
 716             if (
 717                 package_handling.SOL007 == package_type
 718                 or package_handling.SOL007_TOSCA == package_type
 719             ):
 720                 the_package = SOL007Package(package_folder)
 721             elif (
 722                 package_handling.SOL004 == package_type
 723                 or package_handling.SOL004_TOSCA == package_type
 724             ):
 725                 the_package = SOL004Package(package_folder)
 726
 727             the_package.create_or_update_metadata_file()
 728
 729             the_zip_package = shutil.make_archive(
 730                 os.path.join(cwd, package_name),
 731                 "zip",
 732                 os.path.join(directory_name, package_name),
 733             )
 734
 735             print("Package created: {}".format(the_zip_package))
 736
 737             return the_zip_package
 738
 739         except Exception as exc:
 740             raise ClientException(
 741                 "failure during build of zip file (create temp dir, calculate checksum, "
 742                 "zip file): {}".format(exc)
 743             )
 744         finally:
 745             if cwd:
 746                 os.chdir(cwd)
 747             shutil.rmtree(os.path.join(package_folder, "tmp"))
 748
 749     def build_tarfile(self, package_folder, charm_list=None):
 750         """
 751         Creates a .tar.gz file given a package_folder
 752         params: package_folder is the name of the folder to be packaged
 753         returns: .tar.gz name
 754         """
 755         self._logger.debug("")
 756         cwd = None
 757         try:
 758             directory_name, package_name = self.create_temp_dir(
 759                 package_folder, charm_list
 760             )
 761             cwd = os.getcwd()
 762             os.chdir(directory_name)
 763             self.calculate_checksum(package_name)
 764             with tarfile.open("{}.tar.gz".format(package_name), mode="w:gz") as archive:
 765                 print("Adding File: {}".format(package_name))
 766                 archive.add("{}".format(package_name), recursive=True)
 767             # return "Created {}.tar.gz".format(package_folder)
 768             # self.build("{}".format(os.path.basename(package_folder)))
 769             os.chdir(cwd)
 770             cwd = None
 771             created_package = "{}/{}.tar.gz".format(
 772                 os.path.dirname(package_folder) or ".", package_name
 773             )
 774             os.rename(
 775                 "{}/{}.tar.gz".format(directory_name, package_name), created_package
 776             )
 777             os.rename(
 778                 "{}/{}/checksums.txt".format(directory_name, package_name),
 779                 "{}/checksums.txt".format(package_folder),
 780             )
 781             print("Package created: {}".format(created_package))
 782             return created_package
 783         except Exception as exc:
 784             raise ClientException(
 785                 "failure during build of targz file (create temp dir, calculate checksum, "
 786                 "tar.gz file): {}".format(exc)
 787             )
 788         finally:
 789             if cwd:
 790                 os.chdir(cwd)
 791             shutil.rmtree(os.path.join(package_folder, "tmp"))
 792
 793     def create_temp_dir(self, package_folder, charm_list=None):
 794         """
 795         Method to create a temporary folder where we can move the files in package_folder
 796         """
 797         self._logger.debug("")
 798         ignore_patterns = ".gitignore"
 799         ignore = shutil.ignore_patterns(ignore_patterns)
 800         directory_name = os.path.abspath(package_folder)
 801         package_name = os.path.basename(directory_name)
 802         directory_name += "/tmp"
 803         os.makedirs("{}/{}".format(directory_name, package_name), exist_ok=True)
 804         self._logger.debug("Makedirs DONE: {}/{}".format(directory_name, package_name))
 805         for item in os.listdir(package_folder):
 806             self._logger.debug("Item: {}".format(item))
 807             if item != "tmp":
 808                 s = os.path.join(package_folder, item)
 809                 d = os.path.join(os.path.join(directory_name, package_name), item)
 810                 if os.path.isdir(s):
 811                     if item == "charms":
 812                         os.makedirs(d, exist_ok=True)
 813                         s_builds = os.path.join(s, "builds")
 814                         for charm in charm_list:
 815                             self._logger.debug("Copying charm {}".format(charm))
 816                             if charm in os.listdir(s):
 817                                 s_charm = os.path.join(s, charm)
 818                             elif charm in os.listdir(s_builds):
 819                                 s_charm = os.path.join(s_builds, charm)
 820                             else:
 821                                 raise ClientException(
 822                                     "The charm {} referenced in the descriptor file "
 823                                     "could not be found in {}/charms or in {}/charms/builds".format(
 824                                         charm, package_folder, package_folder
 825                                     )
 826                                 )
 827                             d_temp = os.path.join(d, charm)
 828                             self._logger.debug(
 829                                 "Copying tree: {} -> {}".format(s_charm, d_temp)
 830                             )
 831                             if os.path.isdir(s_charm):
 832                                 shutil.copytree(
 833                                     s_charm, d_temp, symlinks=True, ignore=ignore
 834                                 )
 835                             else:
 836                                 shutil.copy2(s_charm, d_temp)
 837                             self._logger.debug("DONE")
 838                     else:
 839                         self._logger.debug("Copying tree: {} -> {}".format(s, d))
 840                         shutil.copytree(s, d, symlinks=True, ignore=ignore)
 841                         self._logger.debug("DONE")
 842                 else:
 843                     if item in ignore_patterns:
 844                         continue
 845                     self._logger.debug("Copying file: {} -> {}".format(s, d))
 846                     shutil.copy2(s, d)
 847                     self._logger.debug("DONE")
 848         return directory_name, package_name
 849
 850     def copy_tree(self, s, d, ignore):
 851         self._logger.debug("Copying tree: {} -> {}".format(s, d))
 852         shutil.copytree(s, d, symlinks=True, ignore=ignore)
 853         self._logger.debug("DONE")
 854
 855     def create_temp_dir_sol004_007(self, package_folder, charm_list=None):
 856         """
 857         Method to create a temporary folder where we can move the files in package_folder
 858         """
 859         self._logger.debug("")
 860         ignore_patterns = ".gitignore"
 861         ignore = shutil.ignore_patterns(ignore_patterns)
 862         directory_name = os.path.abspath(package_folder)
 863         package_name = os.path.basename(directory_name)
 864         directory_name += "/tmp"
 865         os.makedirs("{}/{}".format(directory_name, package_name), exist_ok=True)
 866         self._logger.debug("Makedirs DONE: {}/{}".format(directory_name, package_name))
 867         for item in os.listdir(package_folder):
 868             self._logger.debug("Item: {}".format(item))
 869             if item != "tmp":
 870                 s = os.path.join(package_folder, item)
 871                 d = os.path.join(os.path.join(directory_name, package_name), item)
 872                 if os.path.isdir(s):
 873                     if item == "Scripts":
 874                         os.makedirs(d, exist_ok=True)
 875                         scripts_folder = s
 876                         for script_item in os.listdir(scripts_folder):
 877                             scripts_destination_folder = os.path.join(d, script_item)
 878                             if script_item == "charms":
 879                                 s_builds = os.path.join(
 880                                     scripts_folder, script_item, "builds"
 881                                 )
 882                                 for charm in charm_list:
 883                                     self._logger.debug("Copying charm {}".format(charm))
 884                                     if charm in os.listdir(
 885                                         os.path.join(scripts_folder, script_item)
 886                                     ):
 887                                         s_charm = os.path.join(
 888                                             scripts_folder, script_item, charm
 889                                         )
 890                                     elif charm in os.listdir(s_builds):
 891                                         s_charm = os.path.join(s_builds, charm)
 892                                     else:
 893                                         raise ClientException(
 894                                             "The charm {} referenced in the descriptor file "
 895                                             "could not be found in {}/charms or in {}/charms/builds".format(
 896                                                 charm, package_folder, package_folder
 897                                             )
 898                                         )
 899                                     d_temp = os.path.join(
 900                                         scripts_destination_folder, charm
 901                                     )
 902                                     self.copy_tree(s_charm, d_temp, ignore)
 903                             else:
 904                                 self.copy_tree(
 905                                     os.path.join(scripts_folder, script_item),
 906                                     scripts_destination_folder,
 907                                     ignore,
 908                                 )
 909                     else:
 910                         self.copy_tree(s, d, ignore)
 911                 else:
 912                     if item in ignore_patterns:
 913                         continue
 914                     self._logger.debug("Copying file: {} -> {}".format(s, d))
 915                     shutil.copy2(s, d)
 916                     self._logger.debug("DONE")
 917         return directory_name, package_name
 918
 919     def charms_search(self, descriptor_file, desc_type):
 920         self._logger.debug(
 921             "descriptor_file: {}, desc_type: {}".format(descriptor_file, desc_type)
 922         )
 923         charms_set = set()
 924         with open("{}".format(descriptor_file)) as yaml_desc:
 925             descriptor_dict = yaml.safe_load(yaml_desc)
 926             # self._logger.debug("\n"+yaml.safe_dump(descriptor_dict, indent=4, default_flow_style=False))
 927
 928             if (
 929                 desc_type == "vnf"
 930                 and (
 931                     "vnfd:vnfd-catalog" in descriptor_dict
 932                     or "vnfd-catalog" in descriptor_dict
 933                 )
 934             ) or (
 935                 desc_type == "ns"
 936                 and (
 937                     "nsd:nsd-catalog" in descriptor_dict
 938                     or "nsd-catalog" in descriptor_dict
 939                 )
 940             ):
 941                 charms_set = self._charms_search_on_osm_im_dict(
 942                     descriptor_dict, desc_type
 943                 )
 944             else:
 945                 if desc_type == "ns":
 946                     get_charm_list = self._charms_search_on_nsd_sol006_dict
 947                 elif desc_type == "vnf":
 948                     get_charm_list = self._charms_search_on_vnfd_sol006_dict
 949                 else:
 950                     raise Exception("Bad descriptor type")
 951                 charms_set = get_charm_list(descriptor_dict)
 952         return charms_set
 953
 954     def _charms_search_on_osm_im_dict(self, osm_im_dict, desc_type):
 955         self._logger.debug("")
 956         charms_set = set()
 957         for k1, v1 in osm_im_dict.items():
 958             for k2, v2 in v1.items():
 959                 for entry in v2:
 960                     if "{}-configuration".format(desc_type) in entry:
 961                         vnf_config = entry["{}-configuration".format(desc_type)]
 962                         for k3, v3 in vnf_config.items():
 963                             if "charm" in v3:
 964                                 charms_set.add((v3["charm"]))
 965                     if "vdu" in entry:
 966                         vdus = entry["vdu"]
 967                         for vdu in vdus:
 968                             if "vdu-configuration" in vdu:
 969                                 for k4, v4 in vdu["vdu-configuration"].items():
 970                                     if "charm" in v4:
 971                                         charms_set.add((v4["charm"]))
 972         return charms_set
 973
 974     def _charms_search_on_vnfd_sol006_dict(self, sol006_dict):
 975         self._logger.debug("")
 976         charms_set = set()
 977         dfs = sol006_dict.get("vnfd", {}).get("df", [])
 978         for df in dfs:
 979             day_1_2s = (
 980                 df.get("lcm-operations-configuration", {})
 981                 .get("operate-vnf-op-config", {})
 982                 .get("day1-2")
 983             )
 984             if day_1_2s is not None:
 985                 for day_1_2 in day_1_2s:
 986                     exec_env_list = day_1_2.get("execution-environment-list", [])
 987                     for exec_env in exec_env_list:
 988                         if "juju" in exec_env and "charm" in exec_env["juju"]:
 989                             charms_set.add(exec_env["juju"]["charm"])
 990         return charms_set
 991
 992     def _charms_search_on_nsd_sol006_dict(self, sol006_dict):
 993         self._logger.debug("")
 994         charms_set = set()
 995         nsd_list = sol006_dict.get("nsd", {}).get("nsd", [])
 996         for nsd in nsd_list:
 997             charm = nsd.get("ns-configuration", {}).get("juju", {}).get("charm")
 998             if charm:
 999                 charms_set.add(charm)
1000         return charms_set