osmclient/common/package_tool.py

   1 # /bin/env python3
   2 # Copyright 2019 ATOS
   3 #
   4 # All Rights Reserved.
   5 #
   6 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 #    not use this file except in compliance with the License. You may obtain
   8 #    a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #    Unless required by applicable law or agreed to in writing, software
  13 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 #    License for the specific language governing permissions and limitations
  16 #    under the License.
  17
  18 import glob
  19 import hashlib
  20 import logging
  21 import os
  22 import shutil
  23 import subprocess
  24 import tarfile
  25 import time
  26
  27 from jinja2 import Environment, PackageLoader
  28 from osm_im.validation import Validation as validation_im
  29 from osmclient.common.exceptions import ClientException
  30 import yaml
  31
  32
  33 class PackageTool(object):
  34     def __init__(self, client=None):
  35         self._client = client
  36         self._logger = logging.getLogger('osmclient')
  37
  38     def create(self, package_type, base_directory, package_name, override, image, vdus, vcpu, memory, storage,
  39                interfaces, vendor, detailed, netslice_subnets, netslice_vlds):
  40         """
  41             **Create a package descriptor**
  42
  43             :params:
  44                 - package_type: [vnf, ns, nst]
  45                 - base directory: path of destination folder
  46                 - package_name: is the name of the package to be created
  47                 - image: specify the image of the vdu
  48                 - vcpu: number of virtual cpus of the vdu
  49                 - memory: amount of memory in MB pf the vdu
  50                 - storage: amount of storage in GB of the vdu
  51                 - interfaces: number of interfaces besides management interface
  52                 - vendor: vendor name of the vnf/ns
  53                 - detailed: include all possible values for NSD, VNFD, NST
  54                 - netslice_subnets: number of netslice_subnets for the NST
  55                 - netslice_vlds: number of virtual link descriptors for the NST
  56
  57             :return: status
  58         """
  59         self._logger.debug("")
  60         # print("location: {}".format(osmclient.__path__))
  61         file_loader = PackageLoader("osmclient")
  62         env = Environment(loader=file_loader)
  63         if package_type == 'ns':
  64             template = env.get_template('nsd.yaml.j2')
  65             content = {"name": package_name, "vendor": vendor, "vdus": vdus, "clean": False, "interfaces": interfaces,
  66                        "detailed": detailed}
  67         elif package_type == 'vnf':
  68             template = env.get_template('vnfd.yaml.j2')
  69             content = {"name": package_name, "vendor": vendor, "vdus": vdus, "clean": False, "interfaces": interfaces,
  70                        "image": image, "vcpu": vcpu, "memory": memory, "storage": storage, "detailed": detailed}
  71         elif package_type == 'nst':
  72             template = env.get_template('nst.yaml.j2')
  73             content = {"name": package_name, "vendor": vendor, "interfaces": interfaces,
  74                        "netslice_subnets": netslice_subnets, "netslice_vlds": netslice_vlds, "detailed": detailed}
  75         else:
  76             raise ClientException("Wrong descriptor type {}. Options: ns, vnf, nst".format(package_type))
  77
  78         # print("To be rendered: {}".format(content))
  79         output = template.render(content)
  80         # print(output)
  81
  82         structure = self.discover_folder_structure(base_directory, package_name, override)
  83         if structure.get("folders"):
  84             self.create_folders(structure["folders"], package_type)
  85         if structure.get("files"):
  86             self.create_files(structure["files"], output, package_type)
  87         return "Created"
  88
  89     def validate(self, base_directory, recursive=True):
  90         """
  91             **Validate OSM Descriptors given a path**
  92
  93             :params:
  94                 - base_directory is the root path for all descriptors
  95
  96             :return: List of dict of validated descriptors. keys: type, path, valid, error
  97         """
  98         self._logger.debug("")
  99         table = []
 100         if recursive:
 101             descriptors_paths = [f for f in glob.glob(base_directory + "/**/*.yaml", recursive=recursive)]
 102         else:
 103             descriptors_paths = [f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)]
 104         print("Base directory: {}".format(base_directory))
 105         print("{} Descriptors found to validate".format(len(descriptors_paths)))
 106         for desc_path in descriptors_paths:
 107             with open(desc_path) as descriptor_file:
 108                 descriptor_data = descriptor_file.read()
 109             desc_type = "-"
 110             try:
 111                 desc_type, descriptor_data = validation_im.yaml_validation(self, descriptor_data)
 112                 validation_im.pyangbind_validation(self, desc_type, descriptor_data)
 113                 table.append({"type": desc_type, "path": desc_path, "valid": "OK", "error": "-"})
 114             except Exception as e:
 115                 table.append({"type": desc_type, "path": desc_path, "valid": "ERROR", "error": str(e)})
 116         return table
 117
 118     def build(self, package_folder, skip_validation=False, skip_charm_build=False):
 119         """
 120             **Creates a .tar.gz file given a package_folder**
 121
 122             :params:
 123                 - package_folder: is the name of the folder to be packaged
 124                 - skip_validation: is the flag to validate or not the descriptors on the folder before build
 125
 126             :returns: message result for the build process
 127         """
 128         self._logger.debug("")
 129         package_folder = package_folder.rstrip('/')
 130         if not os.path.exists("{}".format(package_folder)):
 131             return "Fail, package is not in the specified path"
 132         if not skip_validation:
 133             print('Validating package {}'.format(package_folder))
 134             results = self.validate(package_folder, recursive=False)
 135             if results:
 136                 for result in results:
 137                     if result["valid"] != "OK":
 138                         raise ClientException("There was an error validating the file {} with error: {}"
 139                                               .format(result["path"], result["error"]))
 140                 print('Validation OK')
 141             else:
 142                 raise ClientException("No descriptor file found in: {}".format(package_folder))
 143         charm_list = self.build_all_charms(package_folder, skip_charm_build)
 144         return self.build_tarfile(package_folder, charm_list)
 145
 146     def calculate_checksum(self, package_folder):
 147         """
 148             **Function to calculate the checksum given a folder**
 149
 150             :params:
 151                 - package_folder: is the folder where we have the files to calculate the checksum
 152             :returns: None
 153         """
 154         self._logger.debug("")
 155         files = [f for f in glob.glob(package_folder + "/**/*.*", recursive=True) if os.path.isfile(f)]
 156         with open("{}/checksums.txt".format(package_folder), "w+") as checksum:
 157             for file_item in files:
 158                 if "checksums.txt" in file_item:
 159                     continue
 160                 # from https://www.quickprogrammingtips.com/python/how-to-calculate-md5-hash-of-a-file-in-python.html
 161                 md5_hash = hashlib.md5()
 162                 with open(file_item, "rb") as f:
 163                     # Read and update hash in chunks of 4K
 164                     for byte_block in iter(lambda: f.read(4096), b""):
 165                         md5_hash.update(byte_block)
 166                     checksum.write("{}\t{}\n".format(md5_hash.hexdigest(), file_item))
 167
 168     def create_folders(self, folders, package_type):
 169         """
 170             **Create folder given a list of folders**
 171
 172             :params:
 173                 - folders: [List] list of folders paths to be created
 174                 - package_type: is the type of package to be created
 175             :return: None
 176         """
 177         self._logger.debug("")
 178         for folder in folders:
 179             try:
 180                 # print("Folder {} == package_type {}".format(folder[1], package_type))
 181                 if folder[1] == package_type:
 182                     print("Creating folder:\t{}".format(folder[0]))
 183                     os.makedirs(folder[0])
 184             except FileExistsError:
 185                 pass
 186
 187     def save_file(self, file_name, file_body):
 188         """
 189             **Create a file given a name and the content**
 190
 191             :params:
 192                 - file_name: is the name of the file with the relative route
 193                 - file_body: is the content of the file
 194             :return: None
 195         """
 196         self._logger.debug("")
 197         print("Creating file:  \t{}".format(file_name))
 198         try:
 199             with open(file_name, "w+") as f:
 200                 f.write(file_body)
 201         except Exception as e:
 202             raise ClientException(e)
 203
 204     def generate_readme(self):
 205         """
 206             **Creates the README content**
 207
 208             :returns: readme content
 209         """
 210         self._logger.debug("")
 211         return """# Descriptor created by OSM descriptor package generated\n\n**Created on {} **""".format(
 212                time.strftime("%m/%d/%Y, %H:%M:%S", time.localtime()))
 213
 214     def generate_cloud_init(self):
 215         """
 216             **Creates the cloud-init content**
 217
 218             :returns: cloud-init content
 219         """
 220         return "---\n#cloud-config"
 221
 222     def create_files(self, files, file_content, package_type):
 223         """
 224             **Creates the files given the file list and type**
 225
 226             :params:
 227                 - files: is the list of files structure
 228                 - file_content: is the content of the descriptor rendered by the template
 229                 - package_type: is the type of package to filter the creation structure
 230
 231             :return: None
 232         """
 233         self._logger.debug("")
 234         for file_item, file_package, file_type in files:
 235             if package_type == file_package:
 236                 if file_type == "descriptor":
 237                     self.save_file(file_item, file_content)
 238                 elif file_type == "readme":
 239                     self.save_file(file_item, self.generate_readme())
 240                 elif file_type == "cloud_init":
 241                     self.save_file(file_item, self.generate_cloud_init())
 242
 243     def check_files_folders(self, path_list, override):
 244         """
 245             **Find files and folders missing given a directory structure {"folders": [], "files": []}**
 246
 247             :params:
 248                 - path_list: is the list of files and folders to be created
 249                 - override: is the flag used to indicate the creation of the list even if the file exist to override it
 250
 251             :return: Missing paths Dict
 252         """
 253         self._logger.debug("")
 254         missing_paths = {}
 255         folders = []
 256         files = []
 257         for folder in path_list.get("folders"):
 258             if not os.path.exists(folder[0]):
 259                 folders.append(folder)
 260         missing_paths["folders"] = folders
 261
 262         for file_item in path_list.get("files"):
 263             if not os.path.exists(file_item[0]) or override is True:
 264                 files.append(file_item)
 265         missing_paths["files"] = files
 266
 267         return missing_paths
 268
 269     def build_all_charms(self, package_folder, skip_charm_build):
 270         """
 271             **Read the descriptor file, check that the charms referenced are in the folder and compiles them**
 272
 273             :params:
 274                 - packet_folder: is the location of the package
 275             :return: Files and Folders not found. In case of override, it will return all file list
 276         """
 277         self._logger.debug("")
 278         listCharms = []
 279         descriptor_file = False
 280         descriptors_paths = [f for f in glob.glob(package_folder + "/*.yaml")]
 281         for file in descriptors_paths:
 282             if file.endswith('nfd.yaml'):
 283                 descriptor_file = True
 284                 listCharms = self.charms_search(file, 'vnf')
 285             if file.endswith('nsd.yaml'):
 286                 descriptor_file = True
 287                 listCharms = self.charms_search(file, 'ns')
 288         print("List of charms in the descriptor: {}".format(listCharms))
 289         if not descriptor_file:
 290             raise ClientException('descriptor name is not correct in: {}'.format(package_folder))
 291         if listCharms and not skip_charm_build:
 292             for charmName in listCharms:
 293                 if os.path.isdir('{}/charms/layers/{}'.format(package_folder, charmName)):
 294                     print('Building charm {}/charms/layers/{}'.format(package_folder, charmName))
 295                     self.charm_build(package_folder, charmName)
 296                     print('Charm built: {}'.format(charmName))
 297                 else:
 298                     if not os.path.isdir('{}/charms/{}'.format(package_folder, charmName)):
 299                         raise ClientException('The charm: {} referenced in the descriptor file '
 300                                               'is not present either in {}/charms or in {}/charms/layers'.
 301                                               format(charmName, package_folder, package_folder))
 302         self._logger.debug("Return list of charms: {}".format(listCharms))
 303         return listCharms
 304
 305     def discover_folder_structure(self, base_directory, name, override):
 306         """
 307             **Discover files and folders structure for OSM descriptors given a base_directory and name**
 308
 309             :params:
 310                 - base_directory: is the location of the package to be created
 311                 - name: is the name of the package
 312                 - override: is the flag used to indicate the creation of the list even if the file exist to override it
 313             :return: Files and Folders not found. In case of override, it will return all file list
 314         """
 315         self._logger.debug("")
 316         prefix = "{}/{}".format(base_directory, name)
 317         files_folders = {"folders": [("{}_ns".format(prefix), "ns"),
 318                                      ("{}_ns/icons".format(prefix), "ns"),
 319                                      ("{}_ns/charms".format(prefix), "ns"),
 320                                      ("{}_vnf".format(name), "vnf"),
 321                                      ("{}_vnf/charms".format(prefix), "vnf"),
 322                                      ("{}_vnf/cloud_init".format(prefix), "vnf"),
 323                                      ("{}_vnf/images".format(prefix), "vnf"),
 324                                      ("{}_vnf/icons".format(prefix), "vnf"),
 325                                      ("{}_vnf/scripts".format(prefix), "vnf"),
 326                                      ("{}_nst".format(prefix), "nst"),
 327                                      ("{}_nst/icons".format(prefix), "nst")
 328                                      ],
 329                          "files": [("{}_ns/{}_nsd.yaml".format(prefix, name), "ns", "descriptor"),
 330                                    ("{}_ns/README.md".format(prefix), "ns", "readme"),
 331                                    ("{}_vnf/{}_vnfd.yaml".format(prefix, name), "vnf", "descriptor"),
 332                                    ("{}_vnf/cloud_init/cloud-config.txt".format(prefix), "vnf", "cloud_init"),
 333                                    ("{}_vnf/README.md".format(prefix), "vnf", "readme"),
 334                                    ("{}_nst/{}_nst.yaml".format(prefix, name), "nst", "descriptor"),
 335                                    ("{}_nst/README.md".format(prefix), "nst", "readme")
 336                                    ]
 337                          }
 338         missing_files_folders = self.check_files_folders(files_folders, override)
 339         # print("Missing files and folders: {}".format(missing_files_folders))
 340         return missing_files_folders
 341
 342     def charm_build(self, charms_folder, build_name):
 343         """
 344         Build the charms inside the package.
 345         params: package_folder is the name of the folder where is the charms to compile.
 346                 build_name is the name of the layer or interface
 347         """
 348         self._logger.debug("")
 349         os.environ['JUJU_REPOSITORY'] = "{}/charms".format(charms_folder)
 350         os.environ['CHARM_LAYERS_DIR'] = "{}/layers".format(os.environ['JUJU_REPOSITORY'])
 351         os.environ['CHARM_INTERFACES_DIR'] = "{}/interfaces".format(os.environ['JUJU_REPOSITORY'])
 352         os.environ['CHARM_BUILD_DIR'] = "{}/charms/builds".format(charms_folder)
 353         if not os.path.exists(os.environ['CHARM_BUILD_DIR']):
 354             os.makedirs(os.environ['CHARM_BUILD_DIR'])
 355         src_folder = '{}/{}'.format(os.environ['CHARM_LAYERS_DIR'], build_name)
 356         result = subprocess.run(["charm", "build", "{}".format(src_folder)])
 357         if result.returncode == 1:
 358             raise ClientException("failed to build the charm: {}".format(src_folder))
 359         self._logger.verbose("charm {} built".format(src_folder))
 360
 361     def build_tarfile(self, package_folder, charm_list=None):
 362         """
 363         Creates a .tar.gz file given a package_folder
 364         params: package_folder is the name of the folder to be packaged
 365         returns: .tar.gz name
 366         """
 367         self._logger.debug("")
 368         cwd = None
 369         try:
 370             directory_name, package_name = self.create_temp_dir(package_folder, charm_list)
 371             cwd = os.getcwd()
 372             os.chdir(directory_name)
 373             self.calculate_checksum(package_name)
 374             with tarfile.open("{}.tar.gz".format(package_name), mode='w:gz') as archive:
 375                 print("Adding File: {}".format(package_name))
 376                 archive.add('{}'.format(package_name), recursive=True)
 377             # return "Created {}.tar.gz".format(package_folder)
 378             # self.build("{}".format(os.path.basename(package_folder)))
 379             os.chdir(cwd)
 380             cwd = None
 381             created_package = "{}/{}.tar.gz".format(os.path.dirname(package_folder) or '.', package_name)
 382             os.rename("{}/{}.tar.gz".format(directory_name, package_name),
 383                       created_package)
 384             os.rename("{}/{}/checksums.txt".format(directory_name, package_name),
 385                       "{}/checksums.txt".format(package_folder))
 386             print("Package created: {}".format(created_package))
 387             return created_package
 388         except Exception as exc:
 389             raise ClientException('failure during build of targz file (create temp dir, calculate checksum, '
 390                                   'tar.gz file): {}'.format(exc))
 391         finally:
 392             if cwd:
 393                 os.chdir(cwd)
 394             shutil.rmtree(os.path.join(package_folder, "tmp"))
 395
 396     def create_temp_dir(self, package_folder, charm_list=None):
 397         """
 398         Method to create a temporary folder where we can move the files in package_folder
 399         """
 400         self._logger.debug("")
 401         ignore_patterns = ('.gitignore')
 402         ignore = shutil.ignore_patterns(ignore_patterns)
 403         directory_name = os.path.abspath(package_folder)
 404         package_name = os.path.basename(directory_name)
 405         directory_name += "/tmp"
 406         os.makedirs("{}/{}".format(directory_name, package_name), exist_ok=True)
 407         self._logger.debug("Makedirs DONE: {}/{}".format(directory_name, package_name))
 408         for item in os.listdir(package_folder):
 409             self._logger.debug("Item: {}".format(item))
 410             if item != "tmp":
 411                 s = os.path.join(package_folder, item)
 412                 d = os.path.join(os.path.join(directory_name, package_name), item)
 413                 if os.path.isdir(s):
 414                     if item == "charms":
 415                         os.makedirs(d, exist_ok=True)
 416                         s_builds = os.path.join(s, "builds")
 417                         for charm in charm_list:
 418                             self._logger.debug("Copying charm {}".format(charm))
 419                             if charm in os.listdir(s):
 420                                 s_charm = os.path.join(s, charm)
 421                             elif charm in os.listdir(s_builds):
 422                                 s_charm = os.path.join(s_builds, charm)
 423                             else:
 424                                 raise ClientException('The charm {} referenced in the descriptor file '
 425                                                       'could not be found in {}/charms or in {}/charms/builds'.
 426                                                       format(charm, package_folder, package_folder))
 427                             d_temp = os.path.join(d, charm)
 428                             self._logger.debug("Copying tree: {} -> {}".format(s_charm, d_temp))
 429                             shutil.copytree(s_charm, d_temp, symlinks=True, ignore=ignore)
 430                             self._logger.debug("DONE")
 431                     else:
 432                         self._logger.debug("Copying tree: {} -> {}".format(s, d))
 433                         shutil.copytree(s, d, symlinks=True, ignore=ignore)
 434                         self._logger.debug("DONE")
 435                 else:
 436                     if item in ignore_patterns:
 437                         continue
 438                     self._logger.debug("Copying file: {} -> {}".format(s, d))
 439                     shutil.copy2(s, d)
 440                     self._logger.debug("DONE")
 441         return directory_name, package_name
 442
 443     def charms_search(self, descriptor_file, desc_type):
 444         self._logger.debug("descriptor_file: {}, desc_type: {}".format(descriptor_file,
 445                                                                        desc_type))
 446         with open("{}".format(descriptor_file)) as yaml_desc:
 447             descriptor_dict = yaml.safe_load(yaml_desc)
 448             #self._logger.debug("\n"+yaml.safe_dump(descriptor_dict, indent=4, default_flow_style=False))
 449
 450             if ( (desc_type=="vnf" and ("vnfd:vnfd-catalog" in descriptor_dict or "vnfd-catalog" in descriptor_dict)) or
 451                  (desc_type=="ns" and ( "nsd:nsd-catalog" in descriptor_dict or "nsd-catalog" in descriptor_dict)) ):
 452                 charms_list = self._charms_search_on_osm_im_dict(descriptor_dict, desc_type)
 453             else:
 454                 charms_list = self._charms_search_on_sol006_dict(descriptor_dict, desc_type)
 455
 456         return charms_list
 457
 458     def _charms_search_on_osm_im_dict(self, osm_im_dict, desc_type):
 459         self._logger.debug("")
 460         charms_list = []
 461         for k1, v1 in osm_im_dict.items():
 462             for k2, v2 in v1.items():
 463                 for entry in v2:
 464                     if '{}-configuration'.format(desc_type) in entry:
 465                         vnf_config = entry['{}-configuration'.format(desc_type)]
 466                         for k3, v3 in vnf_config.items():
 467                             if 'charm' in v3:
 468                                 charms_list.append((v3['charm']))
 469                     if 'vdu' in entry:
 470                         vdus = entry['vdu']
 471                         for vdu in vdus:
 472                             if 'vdu-configuration' in vdu:
 473                                 for k4, v4 in vdu['vdu-configuration'].items():
 474                                     if 'charm' in v4:
 475                                         charms_list.append((v4['charm']))
 476         return charms_list
 477
 478     def _charms_search_on_sol006_dict(self, sol006_dict, desc_type):
 479         self._logger.debug("")
 480         charms_list = []
 481         for k1, v1 in sol006_dict.items():
 482             for k2, v2 in v1.items():
 483                 if '{}-configuration'.format(desc_type) in k2:
 484                     for vnf_config in v2:
 485                         for k3, v3 in vnf_config.items():
 486                             if 'charm' in v3:
 487                                 charms_list.append((v3['charm']))
 488                 if 'vdu-configuration' in k2:
 489                     for vdu_config in v2:
 490                         for k3, v3 in vdu_config.items():
 491                             if 'charm' in v3:
 492                                 charms_list.append((v3['charm']))
 493         return charms_list