osmclient/common/package_tool.py

   1 # /bin/env python3
   2 # Copyright 2019 ATOS
   3 #
   4 # All Rights Reserved.
   5 #
   6 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 #    not use this file except in compliance with the License. You may obtain
   8 #    a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #    Unless required by applicable law or agreed to in writing, software
  13 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 #    License for the specific language governing permissions and limitations
  16 #    under the License.
  17
  18 import glob
  19 import hashlib
  20 import logging
  21 import os
  22 import shutil
  23 import subprocess
  24 import tarfile
  25 import time
  26
  27 from jinja2 import Environment, PackageLoader
  28 from osm_im.validation import Validation as validation_im
  29 from osmclient.common.exceptions import ClientException
  30 import yaml
  31
  32
  33 class PackageTool(object):
  34     def __init__(self, client=None):
  35         self._client = client
  36         self._logger = logging.getLogger('osmclient')
  37
  38     def create(self, package_type, base_directory, package_name, override, image, vdus, vcpu, memory, storage,
  39                interfaces, vendor, detailed, netslice_subnets, netslice_vlds):
  40         """
  41             **Create a package descriptor**
  42
  43             :params:
  44                 - package_type: [vnf, ns, nst]
  45                 - base directory: path of destination folder
  46                 - package_name: is the name of the package to be created
  47                 - image: specify the image of the vdu
  48                 - vcpu: number of virtual cpus of the vdu
  49                 - memory: amount of memory in MB pf the vdu
  50                 - storage: amount of storage in GB of the vdu
  51                 - interfaces: number of interfaces besides management interface
  52                 - vendor: vendor name of the vnf/ns
  53                 - detailed: include all possible values for NSD, VNFD, NST
  54                 - netslice_subnets: number of netslice_subnets for the NST
  55                 - netslice_vlds: number of virtual link descriptors for the NST
  56
  57             :return: status
  58         """
  59         self._logger.debug("")
  60         # print("location: {}".format(osmclient.__path__))
  61         file_loader = PackageLoader("osmclient")
  62         env = Environment(loader=file_loader)
  63         if package_type == 'ns':
  64             template = env.get_template('nsd.yaml.j2')
  65             content = {"name": package_name, "vendor": vendor, "vdus": vdus, "clean": False, "interfaces": interfaces,
  66                        "detailed": detailed}
  67         elif package_type == 'vnf':
  68             template = env.get_template('vnfd.yaml.j2')
  69             content = {"name": package_name, "vendor": vendor, "vdus": vdus, "clean": False, "interfaces": interfaces,
  70                        "image": image, "vcpu": vcpu, "memory": memory, "storage": storage, "detailed": detailed}
  71         elif package_type == 'nst':
  72             template = env.get_template('nst.yaml.j2')
  73             content = {"name": package_name, "vendor": vendor, "interfaces": interfaces,
  74                        "netslice_subnets": netslice_subnets, "netslice_vlds": netslice_vlds, "detailed": detailed}
  75         else:
  76             raise ClientException("Wrong descriptor type {}. Options: ns, vnf, nst".format(package_type))
  77
  78         # print("To be rendered: {}".format(content))
  79         output = template.render(content)
  80         # print(output)
  81
  82         structure = self.discover_folder_structure(base_directory, package_name, override)
  83         if structure.get("folders"):
  84             self.create_folders(structure["folders"], package_type)
  85         if structure.get("files"):
  86             self.create_files(structure["files"], output, package_type)
  87         return "Created"
  88
  89     def validate(self, base_directory, recursive=True):
  90         """
  91             **Validate OSM Descriptors given a path**
  92
  93             :params:
  94                 - base_directory is the root path for all descriptors
  95
  96             :return: List of dict of validated descriptors. keys: type, path, valid, error
  97         """
  98         self._logger.debug("")
  99         table = []
 100         if recursive:
 101             descriptors_paths = [f for f in glob.glob(base_directory + "/**/*.yaml", recursive=recursive)]
 102         else:
 103             descriptors_paths = [f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)]
 104         print("Base directory: {}".format(base_directory))
 105         print("{} Descriptors found to validate".format(len(descriptors_paths)))
 106         for desc_path in descriptors_paths:
 107             with open(desc_path) as descriptor_file:
 108                 descriptor_data = descriptor_file.read()
 109             desc_type = "-"
 110             try:
 111                 desc_type, descriptor_data = validation_im.yaml_validation(self, descriptor_data)
 112                 validation_im.pyangbind_validation(self, desc_type, descriptor_data)
 113                 table.append({"type": desc_type, "path": desc_path, "valid": "OK", "error": "-"})
 114             except Exception as e:
 115                 table.append({"type": desc_type, "path": desc_path, "valid": "ERROR", "error": str(e)})
 116         return table
 117
 118     def build(self, package_folder, skip_validation=False, skip_charm_build=False):
 119         """
 120             **Creates a .tar.gz file given a package_folder**
 121
 122             :params:
 123                 - package_folder: is the name of the folder to be packaged
 124                 - skip_validation: is the flag to validate or not the descriptors on the folder before build
 125
 126             :returns: message result for the build process
 127         """
 128         self._logger.debug("")
 129         package_folder = package_folder.rstrip('/')
 130         if not os.path.exists("{}".format(package_folder)):
 131             return "Fail, package is not in the specified path"
 132         if not skip_validation:
 133             print('Validating package {}'.format(package_folder))
 134             results = self.validate(package_folder, recursive=False)
 135             if results:
 136                 for result in results:
 137                     if result["valid"] != "OK":
 138                         raise ClientException("There was an error validating the file {} with error: {}"
 139                                               .format(result["path"], result["error"]))
 140                 print('Validation OK')
 141             else:
 142                 raise ClientException("No descriptor file found in: {}".format(package_folder))
 143         charm_list = self.build_all_charms(package_folder, skip_charm_build)
 144         return self.build_tarfile(package_folder, charm_list)
 145
 146     def calculate_checksum(self, package_folder):
 147         """
 148             **Function to calculate the checksum given a folder**
 149
 150             :params:
 151                 - package_folder: is the folder where we have the files to calculate the checksum
 152             :returns: None
 153         """
 154         self._logger.debug("")
 155         files = [f for f in glob.glob(package_folder + "/**/*.*", recursive=True) if os.path.isfile(f)]
 156         with open("{}/checksums.txt".format(package_folder), "w+") as checksum:
 157             for file_item in files:
 158                 if "checksums.txt" in file_item:
 159                     continue
 160                 # from https://www.quickprogrammingtips.com/python/how-to-calculate-md5-hash-of-a-file-in-python.html
 161                 md5_hash = hashlib.md5()
 162                 with open(file_item, "rb") as f:
 163                     # Read and update hash in chunks of 4K
 164                     for byte_block in iter(lambda: f.read(4096), b""):
 165                         md5_hash.update(byte_block)
 166                     checksum.write("{}\t{}\n".format(md5_hash.hexdigest(), file_item))
 167
 168     def create_folders(self, folders, package_type):
 169         """
 170             **Create folder given a list of folders**
 171
 172             :params:
 173                 - folders: [List] list of folders paths to be created
 174                 - package_type: is the type of package to be created
 175             :return: None
 176         """
 177
 178         for folder in folders:
 179             try:
 180                 # print("Folder {} == package_type {}".format(folder[1], package_type))
 181                 if folder[1] == package_type:
 182                     print("Creating folder:\t{}".format(folder[0]))
 183                     os.makedirs(folder[0])
 184             except FileExistsError:
 185                 pass
 186
 187     def save_file(self, file_name, file_body):
 188         """
 189             **Create a file given a name and the content**
 190
 191             :params:
 192                 - file_name: is the name of the file with the relative route
 193                 - file_body: is the content of the file
 194             :return: None
 195         """
 196         print("Creating file:  \t{}".format(file_name))
 197         try:
 198             with open(file_name, "w+") as f:
 199                 f.write(file_body)
 200         except Exception as e:
 201             raise ClientException(e)
 202
 203     def generate_readme(self):
 204         """
 205             **Creates the README content**
 206
 207             :returns: readme content
 208         """
 209         return """# Descriptor created by OSM descriptor package generated\n\n**Created on {} **""".format(
 210                time.strftime("%m/%d/%Y, %H:%M:%S", time.localtime()))
 211
 212     def generate_cloud_init(self):
 213         """
 214             **Creates the cloud-init content**
 215
 216             :returns: cloud-init content
 217         """
 218         return "---\n#cloud-config"
 219
 220     def create_files(self, files, file_content, package_type):
 221         """
 222             **Creates the files given the file list and type**
 223
 224             :params:
 225                 - files: is the list of files structure
 226                 - file_content: is the content of the descriptor rendered by the template
 227                 - package_type: is the type of package to filter the creation structure
 228
 229             :return: None
 230         """
 231         self._logger.debug("")
 232         for file_item, file_package, file_type in files:
 233             if package_type == file_package:
 234                 if file_type == "descriptor":
 235                     self.save_file(file_item, file_content)
 236                 elif file_type == "readme":
 237                     self.save_file(file_item, self.generate_readme())
 238                 elif file_type == "cloud_init":
 239                     self.save_file(file_item, self.generate_cloud_init())
 240
 241     def check_files_folders(self, path_list, override):
 242         """
 243             **Find files and folders missing given a directory structure {"folders": [], "files": []}**
 244
 245             :params:
 246                 - path_list: is the list of files and folders to be created
 247                 - override: is the flag used to indicate the creation of the list even if the file exist to override it
 248
 249             :return: Missing paths Dict
 250         """
 251         self._logger.debug("")
 252         missing_paths = {}
 253         folders = []
 254         files = []
 255         for folder in path_list.get("folders"):
 256             if not os.path.exists(folder[0]):
 257                 folders.append(folder)
 258         missing_paths["folders"] = folders
 259
 260         for file_item in path_list.get("files"):
 261             if not os.path.exists(file_item[0]) or override is True:
 262                 files.append(file_item)
 263         missing_paths["files"] = files
 264
 265         return missing_paths
 266
 267     def build_all_charms(self, package_folder, skip_charm_build):
 268         """
 269             **Read the descriptor file, check that the charms referenced are in the folder and compiles them**
 270
 271             :params:
 272                 - packet_folder: is the location of the package
 273             :return: Files and Folders not found. In case of override, it will return all file list
 274         """
 275         self._logger.debug("")
 276         listCharms = []
 277         descriptor_file = False
 278         descriptors_paths = [f for f in glob.glob(package_folder + "/*.yaml")]
 279         for file in descriptors_paths:
 280             if file.endswith('nfd.yaml'):
 281                 descriptor_file = True
 282                 listCharms = self.charms_search(file, 'vnf')
 283             if file.endswith('nsd.yaml'):
 284                 descriptor_file = True
 285                 listCharms = self.charms_search(file, 'ns')
 286         print("List of charms in the descriptor: {}".format(listCharms))
 287         if not descriptor_file:
 288             raise ClientException('descriptor name is not correct in: {}'.format(package_folder))
 289         if listCharms and not skip_charm_build:
 290             for charmName in listCharms:
 291                 if os.path.isdir('{}/charms/layers/{}'.format(package_folder, charmName)):
 292                     print('Building charm {}/charms/layers/{}'.format(package_folder, charmName))
 293                     self.charm_build(package_folder, charmName)
 294                     print('Charm built: {}'.format(charmName))
 295                 else:
 296                     if not os.path.isdir('{}/charms/{}'.format(package_folder, charmName)):
 297                         raise ClientException('The charm: {} referenced in the descriptor file '
 298                                               'is not present either in {}/charms or in {}/charms/layers'.
 299                                               format(charmName, package_folder, package_folder))
 300         self._logger.debug("Return list of charms: {}".format(listCharms))
 301         return listCharms
 302
 303     def discover_folder_structure(self, base_directory, name, override):
 304         """
 305             **Discover files and folders structure for OSM descriptors given a base_directory and name**
 306
 307             :params:
 308                 - base_directory: is the location of the package to be created
 309                 - name: is the name of the package
 310                 - override: is the flag used to indicate the creation of the list even if the file exist to override it
 311             :return: Files and Folders not found. In case of override, it will return all file list
 312         """
 313         self._logger.debug("")
 314         prefix = "{}/{}".format(base_directory, name)
 315         files_folders = {"folders": [("{}_ns".format(prefix), "ns"),
 316                                      ("{}_ns/icons".format(prefix), "ns"),
 317                                      ("{}_ns/charms".format(prefix), "ns"),
 318                                      ("{}_vnf".format(name), "vnf"),
 319                                      ("{}_vnf/charms".format(prefix), "vnf"),
 320                                      ("{}_vnf/cloud_init".format(prefix), "vnf"),
 321                                      ("{}_vnf/images".format(prefix), "vnf"),
 322                                      ("{}_vnf/icons".format(prefix), "vnf"),
 323                                      ("{}_vnf/scripts".format(prefix), "vnf"),
 324                                      ("{}_nst".format(prefix), "nst"),
 325                                      ("{}_nst/icons".format(prefix), "nst")
 326                                      ],
 327                          "files": [("{}_ns/{}_nsd.yaml".format(prefix, name), "ns", "descriptor"),
 328                                    ("{}_ns/README.md".format(prefix), "ns", "readme"),
 329                                    ("{}_vnf/{}_vnfd.yaml".format(prefix, name), "vnf", "descriptor"),
 330                                    ("{}_vnf/cloud_init/cloud-config.txt".format(prefix), "vnf", "cloud_init"),
 331                                    ("{}_vnf/README.md".format(prefix), "vnf", "readme"),
 332                                    ("{}_nst/{}_nst.yaml".format(prefix, name), "nst", "descriptor"),
 333                                    ("{}_nst/README.md".format(prefix), "nst", "readme")
 334                                    ]
 335                          }
 336         missing_files_folders = self.check_files_folders(files_folders, override)
 337         # print("Missing files and folders: {}".format(missing_files_folders))
 338         return missing_files_folders
 339
 340     def charm_build(self, charms_folder, build_name):
 341         """
 342         Build the charms inside the package.
 343         params: package_folder is the name of the folder where is the charms to compile.
 344                 build_name is the name of the layer or interface
 345         """
 346         self._logger.debug("")
 347         os.environ['JUJU_REPOSITORY'] = "{}/charms".format(charms_folder)
 348         os.environ['CHARM_LAYERS_DIR'] = "{}/layers".format(os.environ['JUJU_REPOSITORY'])
 349         os.environ['CHARM_INTERFACES_DIR'] = "{}/interfaces".format(os.environ['JUJU_REPOSITORY'])
 350         os.environ['CHARM_BUILD_DIR'] = "{}/charms/builds".format(charms_folder)
 351         if not os.path.exists(os.environ['CHARM_BUILD_DIR']):
 352             os.makedirs(os.environ['CHARM_BUILD_DIR'])
 353         src_folder = '{}/{}'.format(os.environ['CHARM_LAYERS_DIR'], build_name)
 354         result = subprocess.run(["charm", "build", "{}".format(src_folder)])
 355         if result.returncode == 1:
 356             raise ClientException("failed to build the charm: {}".format(src_folder))
 357         self._logger.verbose("charm {} built".format(src_folder))
 358
 359     def build_tarfile(self, package_folder, charm_list=None):
 360         """
 361         Creates a .tar.gz file given a package_folder
 362         params: package_folder is the name of the folder to be packaged
 363         returns: .tar.gz name
 364         """
 365         self._logger.debug("")
 366         cwd = None
 367         try:
 368             directory_name, package_name = self.create_temp_dir(package_folder, charm_list)
 369             cwd = os.getcwd()
 370             os.chdir(directory_name)
 371             self.calculate_checksum(package_name)
 372             with tarfile.open("{}.tar.gz".format(package_name), mode='w:gz') as archive:
 373                 print("Adding File: {}".format(package_name))
 374                 archive.add('{}'.format(package_name), recursive=True)
 375             # return "Created {}.tar.gz".format(package_folder)
 376             # self.build("{}".format(os.path.basename(package_folder)))
 377             os.chdir(cwd)
 378             cwd = None
 379             created_package = "{}/{}.tar.gz".format(os.path.dirname(package_folder) or '.', package_name)
 380             os.rename("{}/{}.tar.gz".format(directory_name, package_name),
 381                       created_package)
 382             os.rename("{}/{}/checksums.txt".format(directory_name, package_name),
 383                       "{}/checksums.txt".format(package_folder))
 384             print("Package created: {}".format(created_package))
 385             return created_package
 386         except Exception as exc:
 387             raise ClientException('failure during build of targz file (create temp dir, calculate checksum, '
 388                                   'tar.gz file): {}'.format(exc))
 389         finally:
 390             if cwd:
 391                 os.chdir(cwd)
 392             shutil.rmtree(os.path.join(package_folder, "tmp"))
 393
 394     def create_temp_dir(self, package_folder, charm_list=None):
 395         """
 396         Method to create a temporary folder where we can move the files in package_folder
 397         """
 398         self._logger.debug("")
 399         ignore_patterns = ('.gitignore')
 400         ignore = shutil.ignore_patterns(ignore_patterns)
 401         directory_name = os.path.abspath(package_folder)
 402         package_name = os.path.basename(directory_name)
 403         directory_name += "/tmp"
 404         os.makedirs("{}/{}".format(directory_name, package_name), exist_ok=True)
 405         self._logger.debug("Makedirs DONE: {}/{}".format(directory_name, package_name))
 406         for item in os.listdir(package_folder):
 407             self._logger.debug("Item: {}".format(item))
 408             if item != "tmp":
 409                 s = os.path.join(package_folder, item)
 410                 d = os.path.join(os.path.join(directory_name, package_name), item)
 411                 if os.path.isdir(s):
 412                     if item == "charms":
 413                         os.makedirs(d, exist_ok=True)
 414                         s_builds = os.path.join(s, "builds")
 415                         for charm in charm_list:
 416                             self._logger.debug("Copying charm {}".format(charm))
 417                             if charm in os.listdir(s):
 418                                 s_charm = os.path.join(s, charm)
 419                             elif charm in os.listdir(s_builds):
 420                                 s_charm = os.path.join(s_builds, charm)
 421                             else:
 422                                 raise ClientException('The charm {} referenced in the descriptor file '
 423                                                       'could not be found in {}/charms or in {}/charms/builds'.
 424                                                       format(charm, package_folder, package_folder))
 425                             d_temp = os.path.join(d, charm)
 426                             self._logger.debug("Copying tree: {} -> {}".format(s_charm, d_temp))
 427                             shutil.copytree(s_charm, d_temp, symlinks=True, ignore=ignore)
 428                             self._logger.debug("DONE")
 429                     else:
 430                         self._logger.debug("Copying tree: {} -> {}".format(s, d))
 431                         shutil.copytree(s, d, symlinks=True, ignore=ignore)
 432                         self._logger.debug("DONE")
 433                 else:
 434                     if item in ignore_patterns:
 435                         continue
 436                     self._logger.debug("Copying file: {} -> {}".format(s, d))
 437                     shutil.copy2(s, d)
 438                     self._logger.debug("DONE")
 439         return directory_name, package_name
 440
 441     def charms_search(self, descriptor_file, desc_type):
 442         self._logger.debug("")
 443         with open("{}".format(descriptor_file)) as yaml_desc:
 444             descriptor_dict = yaml.safe_load(yaml_desc)
 445             if "catalog" in descriptor_dict:  # Match OSM-IM vnfd-catalog and nsd-catalog
 446                 charms_list = self._charms_search_on_sol006_dict(descriptor_dict, desc_type)
 447             else:
 448                 charms_list = self._charms_search_on_osm_im_dict(descriptor_dict, desc_type)
 449
 450         return charms_list
 451
 452     def _charms_search_on_osm_im_dict(self, osm_im_dict, desc_type):
 453         charms_list = []
 454         for k1, v1 in osm_im_dict.items():
 455             for k2, v2 in v1.items():
 456                 for entry in v2:
 457                     if '{}-configuration'.format(desc_type) in entry:
 458                         vnf_config = entry['{}-configuration'.format(desc_type)]
 459                         for k3, v3 in vnf_config.items():
 460                             if 'charm' in v3:
 461                                 charms_list.append((v3['charm']))
 462                     if 'vdu' in entry:
 463                         vdus = entry['vdu']
 464                         for vdu in vdus:
 465                             if 'vdu-configuration' in vdu:
 466                                 for k4, v4 in vdu['vdu-configuration'].items():
 467                                     if 'charm' in v4:
 468                                         charms_list.append((v4['charm']))
 469         return charms_list
 470
 471     def _charms_search_on_sol006_dict(self, sol006_dict, desc_type):
 472         charms_list = []
 473         for k1, v1 in sol006_dict.items():
 474             for k2, v2 in v1.items():
 475                 if '{}-configuration'.format(desc_type) in k2:
 476                     for vnf_config in v2:
 477                         for k3, v3 in vnf_config.items():
 478                             if 'charm' in v3:
 479                                 charms_list.append((v3['charm']))
 480                 if 'vdu-configuration' in k2:
 481                     for vdu_config in v2:
 482                         for k3, v3 in vdu_config.items():
 483                             if 'charm' in v3:
 484                                 charms_list.append((v3['charm']))
 485         return charms_list