osmclient/common/package_tool.py

   1 # /bin/env python3
   2 # Copyright 2019 ATOS
   3 #
   4 # All Rights Reserved.
   5 #
   6 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 #    not use this file except in compliance with the License. You may obtain
   8 #    a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #    Unless required by applicable law or agreed to in writing, software
  13 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 #    License for the specific language governing permissions and limitations
  16 #    under the License.
  17
  18 from osmclient.common.exceptions import ClientException
  19 import os
  20 import glob
  21 import time
  22 import tarfile
  23 import hashlib
  24 from osm_im.validation import Validation as validation_im
  25 from jinja2 import Environment, PackageLoader
  26 import subprocess
  27 import shutil
  28 import yaml
  29 import logging
  30
  31 class PackageTool(object):
  32     def __init__(self, client=None):
  33         self._client = client
  34         self._logger = logging.getLogger('osmclient')
  35
  36     def create(self, package_type, base_directory, package_name, override, image, vdus, vcpu, memory, storage,
  37                interfaces, vendor, detailed, netslice_subnets, netslice_vlds):
  38         """
  39             **Create a package descriptor**
  40
  41             :params:
  42                 - package_type: [vnf, ns, nst]
  43                 - base directory: path of destination folder
  44                 - package_name: is the name of the package to be created
  45                 - image: specify the image of the vdu
  46                 - vcpu: number of virtual cpus of the vdu
  47                 - memory: amount of memory in MB pf the vdu
  48                 - storage: amount of storage in GB of the vdu
  49                 - interfaces: number of interfaces besides management interface
  50                 - vendor: vendor name of the vnf/ns
  51                 - detailed: include all possible values for NSD, VNFD, NST
  52                 - netslice_subnets: number of netslice_subnets for the NST
  53                 - netslice_vlds: number of virtual link descriptors for the NST
  54
  55             :return: status
  56         """
  57         self._logger.debug("")
  58         # print("location: {}".format(osmclient.__path__))
  59         file_loader = PackageLoader("osmclient")
  60         env = Environment(loader=file_loader)
  61         if package_type == 'ns':
  62             template = env.get_template('nsd.yaml.j2')
  63             content = {"name": package_name, "vendor": vendor, "vdus": vdus, "clean": False, "interfaces": interfaces,
  64                        "detailed": detailed}
  65         elif package_type == 'vnf':
  66             template = env.get_template('vnfd.yaml.j2')
  67             content = {"name": package_name, "vendor": vendor, "vdus": vdus, "clean": False, "interfaces": interfaces,
  68                        "image": image, "vcpu": vcpu, "memory": memory, "storage": storage, "detailed": detailed}
  69         elif package_type == 'nst':
  70             template = env.get_template('nst.yaml.j2')
  71             content = {"name": package_name, "vendor": vendor, "interfaces": interfaces,
  72                        "netslice_subnets": netslice_subnets, "netslice_vlds": netslice_vlds, "detailed": detailed}
  73         else:
  74             raise ClientException("Wrong descriptor type {}. Options: ns, vnf, nst".format(package_type))
  75
  76         # print("To be rendered: {}".format(content))
  77         output = template.render(content)
  78         # print(output)
  79
  80         structure = self.discover_folder_structure(base_directory, package_name, override)
  81         if structure.get("folders"):
  82             self.create_folders(structure["folders"], package_type)
  83         if structure.get("files"):
  84             self.create_files(structure["files"], output, package_type)
  85         return "Created"
  86
  87     def validate(self, base_directory, recursive=True):
  88         """
  89             **Validate OSM Descriptors given a path**
  90
  91             :params:
  92                 - base_directory is the root path for all descriptors
  93
  94             :return: List of dict of validated descriptors. keys: type, path, valid, error
  95         """
  96         self._logger.debug("")
  97         table = []
  98         if recursive:
  99             descriptors_paths = [f for f in glob.glob(base_directory + "/**/*.yaml", recursive=recursive)]
 100         else:
 101             descriptors_paths = [f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)]
 102         print("Base directory: {}".format(base_directory))
 103         print("{} Descriptors found to validate".format(len(descriptors_paths)))
 104         for desc_path in descriptors_paths:
 105             with open(desc_path) as descriptor_file:
 106                 descriptor_data = descriptor_file.read()
 107             desc_type = "-"
 108             try:
 109                 desc_type, descriptor_data = validation_im.yaml_validation(self, descriptor_data)
 110                 validation_im.pyangbind_validation(self, desc_type, descriptor_data)
 111                 table.append({"type": desc_type, "path": desc_path, "valid": "OK", "error": "-"})
 112             except Exception as e:
 113                 table.append({"type": desc_type, "path": desc_path, "valid": "ERROR", "error": str(e)})
 114         return table
 115
 116     def build(self, package_folder, skip_validation=False, skip_charm_build=False):
 117         """
 118             **Creates a .tar.gz file given a package_folder**
 119
 120             :params:
 121                 - package_folder: is the name of the folder to be packaged
 122                 - skip_validation: is the flag to validate or not the descriptors on the folder before build
 123
 124             :returns: message result for the build process
 125         """
 126         self._logger.debug("")
 127         package_folder = package_folder.rstrip('/')
 128         if not os.path.exists("{}".format(package_folder)):
 129             return "Fail, package is not in the specified path"
 130         if not skip_validation:
 131             print('Validating package {}'.format(package_folder))
 132             results = self.validate(package_folder, recursive=False)
 133             if results:
 134                 for result in results:
 135                     if result["valid"] != "OK":
 136                         raise ClientException("There was an error validating the file {} with error: {}"
 137                                               .format(result["path"], result["error"]))
 138                 print('Validation OK')
 139             else:
 140                 raise ClientException("No descriptor file found in: {}".format(package_folder))
 141         charm_list = self.build_all_charms(package_folder, skip_charm_build)
 142         return self.build_tarfile(package_folder, charm_list)
 143
 144     def calculate_checksum(self, package_folder):
 145         """
 146             **Function to calculate the checksum given a folder**
 147
 148             :params:
 149                 - package_folder: is the folder where we have the files to calculate the checksum
 150             :returns: None
 151         """
 152         self._logger.debug("")
 153         files = [f for f in glob.glob(package_folder + "/**/*.*", recursive=True)]
 154         checksum = open("{}/checksums.txt".format(package_folder), "w+")
 155         for file_item in files:
 156             if "checksums.txt" in file_item:
 157                 continue
 158             # from https://www.quickprogrammingtips.com/python/how-to-calculate-md5-hash-of-a-file-in-python.html
 159             md5_hash = hashlib.md5()
 160             with open(file_item, "rb") as f:
 161                 # Read and update hash in chunks of 4K
 162                 for byte_block in iter(lambda: f.read(4096), b""):
 163                     md5_hash.update(byte_block)
 164                 checksum.write("{}\t{}\n".format(md5_hash.hexdigest(), file_item))
 165         checksum.close()
 166
 167     def create_folders(self, folders, package_type):
 168         """
 169             **Create folder given a list of folders**
 170
 171             :params:
 172                 - folders: [List] list of folders paths to be created
 173                 - package_type: is the type of package to be created
 174             :return: None
 175         """
 176
 177         for folder in folders:
 178             try:
 179                 # print("Folder {} == package_type {}".format(folder[1], package_type))
 180                 if folder[1] == package_type:
 181                     print("Creating folder:\t{}".format(folder[0]))
 182                     os.makedirs(folder[0])
 183             except FileExistsError:
 184                 pass
 185
 186     def save_file(self, file_name, file_body):
 187         """
 188             **Create a file given a name and the content**
 189
 190             :params:
 191                 - file_name: is the name of the file with the relative route
 192                 - file_body: is the content of the file
 193             :return: None
 194         """
 195         print("Creating file:  \t{}".format(file_name))
 196         try:
 197             with open(file_name, "w+") as f:
 198                 f.write(file_body)
 199         except Exception as e:
 200             raise ClientException(e)
 201
 202     def generate_readme(self):
 203         """
 204             **Creates the README content**
 205
 206             :returns: readme content
 207         """
 208         return """# Descriptor created by OSM descriptor package generated\n\n**Created on {} **""".format(
 209                time.strftime("%m/%d/%Y, %H:%M:%S", time.localtime()))
 210
 211     def generate_cloud_init(self):
 212         """
 213             **Creates the cloud-init content**
 214
 215             :returns: cloud-init content
 216         """
 217         return "---\n#cloud-config"
 218
 219     def create_files(self, files, file_content, package_type):
 220         """
 221             **Creates the files given the file list and type**
 222
 223             :params:
 224                 - files: is the list of files structure
 225                 - file_content: is the content of the descriptor rendered by the template
 226                 - package_type: is the type of package to filter the creation structure
 227
 228             :return: None
 229         """
 230         self._logger.debug("")
 231         for file_item, file_package, file_type in files:
 232             if package_type == file_package:
 233                 if file_type == "descriptor":
 234                     self.save_file(file_item, file_content)
 235                 elif file_type == "readme":
 236                     self.save_file(file_item, self.generate_readme())
 237                 elif file_type == "cloud_init":
 238                     self.save_file(file_item, self.generate_cloud_init())
 239
 240     def check_files_folders(self, path_list, override):
 241         """
 242             **Find files and folders missing given a directory structure {"folders": [], "files": []}**
 243
 244             :params:
 245                 - path_list: is the list of files and folders to be created
 246                 - override: is the flag used to indicate the creation of the list even if the file exist to override it
 247
 248             :return: Missing paths Dict
 249         """
 250         self._logger.debug("")
 251         missing_paths = {}
 252         folders = []
 253         files = []
 254         for folder in path_list.get("folders"):
 255             if not os.path.exists(folder[0]):
 256                 folders.append(folder)
 257         missing_paths["folders"] = folders
 258
 259         for file_item in path_list.get("files"):
 260             if not os.path.exists(file_item[0]) or override is True:
 261                 files.append(file_item)
 262         missing_paths["files"] = files
 263
 264         return missing_paths
 265
 266     def build_all_charms(self, package_folder, skip_charm_build):
 267         """
 268             **Read the descriptor file, check that the charms referenced are in the folder and compiles them**
 269
 270             :params:
 271                 - packet_folder: is the location of the package
 272             :return: Files and Folders not found. In case of override, it will return all file list
 273         """
 274         self._logger.debug("")
 275         listCharms = []
 276         descriptor_file = False
 277         descriptors_paths = [f for f in glob.glob(package_folder + "/*.yaml")]
 278         for file in descriptors_paths:
 279             if file.endswith('nfd.yaml'):
 280                 descriptor_file = True
 281                 listCharms = self.charms_search(file, 'vnf')
 282             if file.endswith('nsd.yaml'):
 283                 descriptor_file = True
 284                 listCharms = self.charms_search(file, 'ns')
 285         print("List of charms in the descriptor: {}".format(listCharms))
 286         if not descriptor_file:
 287             raise ClientException ('descriptor name is not correct in: {}'.format(package_folder))
 288         if listCharms and not skip_charm_build:
 289             for charmName in listCharms:
 290                 if os.path.isdir('{}/charms/layers/{}'.format(package_folder,charmName)):
 291                     print('Building charm {}/charms/layers/{}'.format(package_folder, charmName))
 292                     self.charm_build(package_folder, charmName)
 293                     print('Charm built'.format(charmName))
 294                 else:
 295                     if not os.path.isdir('{}/charms/{}'.format(package_folder,charmName)):
 296                         raise ClientException ('The charm: {} referenced in the descriptor file '
 297                                                'is not present either in {}/charms or in {}/charms/layers'.
 298                                                format(charmName, package_folder,package_folder))
 299         self._logger.debug("Return list of charms: {}".format(listCharms))
 300         return listCharms
 301
 302     def discover_folder_structure(self, base_directory, name, override):
 303         """
 304             **Discover files and folders structure for OSM descriptors given a base_directory and name**
 305
 306             :params:
 307                 - base_directory: is the location of the package to be created
 308                 - name: is the name of the package
 309                 - override: is the flag used to indicate the creation of the list even if the file exist to override it
 310             :return: Files and Folders not found. In case of override, it will return all file list
 311         """
 312         self._logger.debug("")
 313         prefix = "{}/{}".format(base_directory, name)
 314         files_folders = {"folders": [("{}_ns".format(prefix), "ns"),
 315                                      ("{}_ns/icons".format(prefix), "ns"),
 316                                      ("{}_ns/charms".format(prefix), "ns"),
 317                                      ("{}_vnf".format(name), "vnf"),
 318                                      ("{}_vnf/charms".format(prefix), "vnf"),
 319                                      ("{}_vnf/cloud_init".format(prefix), "vnf"),
 320                                      ("{}_vnf/images".format(prefix), "vnf"),
 321                                      ("{}_vnf/icons".format(prefix), "vnf"),
 322                                      ("{}_vnf/scripts".format(prefix), "vnf"),
 323                                      ("{}_nst".format(prefix), "nst"),
 324                                      ("{}_nst/icons".format(prefix), "nst")
 325                                      ],
 326                          "files": [("{}_ns/{}_nsd.yaml".format(prefix, name), "ns", "descriptor"),
 327                                    ("{}_ns/README.md".format(prefix), "ns", "readme"),
 328                                    ("{}_vnf/{}_vnfd.yaml".format(prefix, name), "vnf", "descriptor"),
 329                                    ("{}_vnf/cloud_init/cloud-config.txt".format(prefix), "vnf", "cloud_init"),
 330                                    ("{}_vnf/README.md".format(prefix), "vnf", "readme"),
 331                                    ("{}_nst/{}_nst.yaml".format(prefix, name), "nst", "descriptor"),
 332                                    ("{}_nst/README.md".format(prefix), "nst", "readme")
 333                                    ]
 334                          }
 335         missing_files_folders = self.check_files_folders(files_folders, override)
 336         # print("Missing files and folders: {}".format(missing_files_folders))
 337         return missing_files_folders
 338
 339     def charm_build(self, charms_folder, build_name):
 340         """
 341         Build the charms inside the package.
 342         params: package_folder is the name of the folder where is the charms to compile.
 343                 build_name is the name of the layer or interface
 344         """
 345         self._logger.debug("")
 346         os.environ['JUJU_REPOSITORY'] = "{}/charms".format(charms_folder)
 347         os.environ['CHARM_LAYERS_DIR'] = "{}/layers".format(os.environ['JUJU_REPOSITORY'])
 348         os.environ['CHARM_INTERFACES_DIR'] = "{}/interfaces".format(os.environ['JUJU_REPOSITORY'])
 349         os.environ['CHARM_BUILD_DIR'] = "{}/charms/builds".format(charms_folder)
 350         if not os.path.exists(os.environ['CHARM_BUILD_DIR']):
 351             os.makedirs(os.environ['CHARM_BUILD_DIR'])
 352         src_folder = '{}/{}'.format(os.environ['CHARM_LAYERS_DIR'], build_name)
 353         result = subprocess.run(["charm", "build", "{}".format(src_folder)])
 354         if result.returncode == 1:
 355             raise ClientException("failed to build the charm: {}".format(src_folder))
 356         self._logger.verbose("charm {} built".format(src_folder))
 357
 358     def build_tarfile(self, package_folder, charm_list=None):
 359         """
 360         Creates a .tar.gz file given a package_folder
 361         params: package_folder is the name of the folder to be packaged
 362         returns: .tar.gz name
 363         """
 364         self._logger.debug("")
 365         try:
 366             directory_name = self.create_temp_dir(package_folder, charm_list)
 367             cwd = os.getcwd()
 368             os.chdir(directory_name)
 369             self.calculate_checksum(package_folder)
 370             with tarfile.open("{}.tar.gz".format(package_folder), mode='w:gz') as archive:
 371                 print("Adding File: {}".format(package_folder))
 372                 archive.add('{}'.format(package_folder), recursive=True)
 373             #return "Created {}.tar.gz".format(package_folder)
 374             #self.build("{}".format(os.path.basename(package_folder)))
 375             os.chdir(cwd)
 376         except Exception as exc:
 377             shutil.rmtree(os.path.join(package_folder, "tmp"))
 378             raise ClientException('failure during build of targz file (create temp dir, calculate checksum, tar.gz file): {}'.format(exc))
 379         os.rename("{}/{}.tar.gz".format(directory_name, os.path.basename(package_folder)),
 380                   "{}.tar.gz".format(os.path.basename(package_folder)))
 381         os.rename("{}/{}/checksums.txt".format(directory_name, os.path.basename(package_folder)),
 382                   "{}/checksums.txt".format(package_folder))
 383         shutil.rmtree(os.path.join(package_folder, "tmp"))
 384         print("Package created: {}.tar.gz".format(os.path.basename(package_folder)))
 385         return "{}.tar.gz".format(package_folder)
 386
 387     def create_temp_dir(self, package_folder, charm_list=None):
 388         """
 389         Method to create a temporary folder where we can move the files in package_folder
 390         """
 391         self._logger.debug("")
 392         ignore_patterns = ('.gitignore')
 393         ignore = shutil.ignore_patterns(ignore_patterns)
 394         directory_name = os.path.abspath("{}/tmp".format(package_folder))
 395         os.makedirs("{}/{}".format(directory_name, os.path.basename(package_folder),exist_ok=True))
 396         self._logger.debug("Makedirs DONE: {}/{}".format(directory_name, os.path.basename(package_folder)))
 397         for item in os.listdir(package_folder):
 398             self._logger.debug("Item: {}".format(item))
 399             if item != "tmp":
 400                 s = os.path.join(package_folder, item)
 401                 d = os.path.join(os.path.join(directory_name, os.path.basename(package_folder)), item)
 402                 if os.path.isdir(s):
 403                     if item == "charms":
 404                         os.makedirs(d, exist_ok=True)
 405                         s_builds = os.path.join(s, "builds")
 406                         for charm in charm_list:
 407                             self._logger.debug("Copying charm {}".format(charm))
 408                             if charm in os.listdir(s):
 409                                 s_charm = os.path.join(s, charm)
 410                             elif charm in os.listdir(s_builds):
 411                                 s_charm = os.path.join(s_builds, charm)
 412                             else:
 413                                 raise ClientException('The charm {} referenced in the descriptor file '
 414                                                       'could not be found in {}/charms or in {}/charms/builds'.
 415                                                       format(charm, package_folder, package_folder))
 416                             d_temp = os.path.join(d, charm)
 417                             self._logger.debug("Copying tree: {} -> {}".format(s_charm, d_temp))
 418                             shutil.copytree(s_charm, d_temp, symlinks = True, ignore = ignore)
 419                             self._logger.debug("DONE")
 420                     else:
 421                         self._logger.debug("Copying tree: {} -> {}".format(s,d))
 422                         shutil.copytree(s, d, symlinks = True, ignore = ignore)
 423                         self._logger.debug("DONE")
 424                 else:
 425                     if item in ignore_patterns:
 426                         continue
 427                     self._logger.debug("Copying file: {} -> {}".format(s,d))
 428                     shutil.copy2(s, d)
 429                     self._logger.debug("DONE")
 430         return directory_name
 431
 432     def charms_search(self, descriptor_file, desc_type):
 433         self._logger.debug("")
 434         dict = {}
 435         list = []
 436         with open("{}".format(descriptor_file)) as yaml_desc:
 437             dict = yaml.safe_load(yaml_desc)
 438             for k1, v1 in dict.items():
 439                 for k2, v2 in v1.items():
 440                     for entry in v2:
 441                         if '{}-configuration'.format(desc_type) in entry:
 442                             name = entry['{}-configuration'.format(desc_type)]
 443                             for k3, v3 in name.items():
 444                                 if 'charm' in v3:
 445                                     list.append((v3['charm']))
 446                         if 'vdu' in entry:
 447                             name = entry['vdu']
 448                             for vdu in name:
 449                                 if 'vdu-configuration' in vdu:
 450                                     for k4, v4 in vdu['vdu-configuration'].items():
 451                                         if 'charm' in v4:
 452                                             list.append((v4['charm']))
 453         return list
 454