osmclient/common/package_tool.py

   1 # /bin/env python3
   2 # Copyright 2019 ATOS
   3 #
   4 # All Rights Reserved.
   5 #
   6 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 #    not use this file except in compliance with the License. You may obtain
   8 #    a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #    Unless required by applicable law or agreed to in writing, software
  13 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 #    License for the specific language governing permissions and limitations
  16 #    under the License.
  17
  18 from osmclient.common.exceptions import ClientException
  19 import os
  20 import glob
  21 import time
  22 import tarfile
  23 import hashlib
  24 from osm_im.validation import Validation as validation_im
  25 from jinja2 import Environment, PackageLoader
  26 import subprocess
  27 import shutil
  28 import yaml
  29 import logging
  30
  31 class PackageTool(object):
  32     def __init__(self, client=None):
  33         self._client = client
  34         self._logger = logging.getLogger('osmclient')
  35
  36     def create(self, package_type, base_directory, package_name, override, image, vdus, vcpu, memory, storage,
  37                interfaces, vendor, detailed, netslice_subnets, netslice_vlds):
  38         """
  39             **Create a package descriptor**
  40
  41             :params:
  42                 - package_type: [vnf, ns, nst]
  43                 - base directory: path of destination folder
  44                 - package_name: is the name of the package to be created
  45                 - image: specify the image of the vdu
  46                 - vcpu: number of virtual cpus of the vdu
  47                 - memory: amount of memory in MB pf the vdu
  48                 - storage: amount of storage in GB of the vdu
  49                 - interfaces: number of interfaces besides management interface
  50                 - vendor: vendor name of the vnf/ns
  51                 - detailed: include all possible values for NSD, VNFD, NST
  52                 - netslice_subnets: number of netslice_subnets for the NST
  53                 - netslice_vlds: number of virtual link descriptors for the NST
  54
  55             :return: status
  56         """
  57         self._logger.debug("")
  58         # print("location: {}".format(osmclient.__path__))
  59         file_loader = PackageLoader("osmclient")
  60         env = Environment(loader=file_loader)
  61         if package_type == 'ns':
  62             template = env.get_template('nsd.yaml.j2')
  63             content = {"name": package_name, "vendor": vendor, "vdus": vdus, "clean": False, "interfaces": interfaces,
  64                        "detailed": detailed}
  65         elif package_type == 'vnf':
  66             template = env.get_template('vnfd.yaml.j2')
  67             content = {"name": package_name, "vendor": vendor, "vdus": vdus, "clean": False, "interfaces": interfaces,
  68                        "image": image, "vcpu": vcpu, "memory": memory, "storage": storage, "detailed": detailed}
  69         elif package_type == 'nst':
  70             template = env.get_template('nst.yaml.j2')
  71             content = {"name": package_name, "vendor": vendor, "interfaces": interfaces,
  72                        "netslice_subnets": netslice_subnets, "netslice_vlds": netslice_vlds, "detailed": detailed}
  73         else:
  74             raise ClientException("Wrong descriptor type {}. Options: ns, vnf, nst".format(package_type))
  75
  76         # print("To be rendered: {}".format(content))
  77         output = template.render(content)
  78         # print(output)
  79
  80         structure = self.discover_folder_structure(base_directory, package_name, override)
  81         if structure.get("folders"):
  82             self.create_folders(structure["folders"], package_type)
  83         if structure.get("files"):
  84             self.create_files(structure["files"], output, package_type)
  85         return "Created"
  86
  87     def validate(self, base_directory, recursive=True):
  88         """
  89             **Validate OSM Descriptors given a path**
  90
  91             :params:
  92                 - base_directory is the root path for all descriptors
  93
  94             :return: List of dict of validated descriptors. keys: type, path, valid, error
  95         """
  96         self._logger.debug("")
  97         table = []
  98         if recursive:
  99             descriptors_paths = [f for f in glob.glob(base_directory + "/**/*.yaml", recursive=recursive)]
 100         else:
 101             descriptors_paths = [f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)]
 102         print("Base directory: {}".format(base_directory))
 103         print("{} Descriptors found to validate".format(len(descriptors_paths)))
 104         for desc_path in descriptors_paths:
 105             with open(desc_path) as descriptor_file:
 106                 descriptor_data = descriptor_file.read()
 107             desc_type = "-"
 108             try:
 109                 desc_type, descriptor_data = validation_im.yaml_validation(self, descriptor_data)
 110                 validation_im.pyangbind_validation(self, desc_type, descriptor_data)
 111                 table.append({"type": desc_type, "path": desc_path, "valid": "OK", "error": "-"})
 112             except Exception as e:
 113                 table.append({"type": desc_type, "path": desc_path, "valid": "ERROR", "error": str(e)})
 114         return table
 115
 116     def build(self, package_folder, skip_validation=False, skip_charm_build=False):
 117         """
 118             **Creates a .tar.gz file given a package_folder**
 119
 120             :params:
 121                 - package_folder: is the name of the folder to be packaged
 122                 - skip_validation: is the flag to validate or not the descriptors on the folder before build
 123
 124             :returns: message result for the build process
 125         """
 126         self._logger.debug("")
 127         package_folder = package_folder.rstrip('/')
 128         if not os.path.exists("{}".format(package_folder)):
 129             return "Fail, package is not in the specified route"
 130         if not skip_validation:
 131             results = self.validate(package_folder, recursive=False)
 132             if results:
 133                 for result in results:
 134                     if result["valid"] != "OK":
 135                         raise ClientException("There was an error validating the file: {} with error: {}"
 136                                               .format(result["path"], result["error"]))
 137             else:
 138                 raise ClientException("No descriptor file found in: {}".format(package_folder))
 139         charm_list = self.build_all_charms(package_folder, skip_charm_build)
 140         return self.build_tarfile(package_folder, charm_list)
 141
 142     def calculate_checksum(self, package_folder):
 143         """
 144             **Function to calculate the checksum given a folder**
 145
 146             :params:
 147                 - package_folder: is the folder where we have the files to calculate the checksum
 148             :returns: None
 149         """
 150         self._logger.debug("")
 151         files = [f for f in glob.glob(package_folder + "/**/*.*", recursive=True)]
 152         checksum = open("{}/checksums.txt".format(package_folder), "w+")
 153         for file_item in files:
 154             if "checksums.txt" in file_item:
 155                 continue
 156             # from https://www.quickprogrammingtips.com/python/how-to-calculate-md5-hash-of-a-file-in-python.html
 157             md5_hash = hashlib.md5()
 158             with open(file_item, "rb") as f:
 159                 # Read and update hash in chunks of 4K
 160                 for byte_block in iter(lambda: f.read(4096), b""):
 161                     md5_hash.update(byte_block)
 162                 checksum.write("{}\t{}\n".format(md5_hash.hexdigest(), file_item))
 163         checksum.close()
 164
 165     def create_folders(self, folders, package_type):
 166         """
 167             **Create folder given a list of folders**
 168
 169             :params:
 170                 - folders: [List] list of folders paths to be created
 171                 - package_type: is the type of package to be created
 172             :return: None
 173         """
 174
 175         for folder in folders:
 176             try:
 177                 # print("Folder {} == package_type {}".format(folder[1], package_type))
 178                 if folder[1] == package_type:
 179                     print("Creating folder:\t{}".format(folder[0]))
 180                     os.makedirs(folder[0])
 181             except FileExistsError:
 182                 pass
 183
 184     def save_file(self, file_name, file_body):
 185         """
 186             **Create a file given a name and the content**
 187
 188             :params:
 189                 - file_name: is the name of the file with the relative route
 190                 - file_body: is the content of the file
 191             :return: None
 192         """
 193         print("Creating file:  \t{}".format(file_name))
 194         try:
 195             with open(file_name, "w+") as f:
 196                 f.write(file_body)
 197         except Exception as e:
 198             raise ClientException(e)
 199
 200     def generate_readme(self):
 201         """
 202             **Creates the README content**
 203
 204             :returns: readme content
 205         """
 206         return """# Descriptor created by OSM descriptor package generated\n\n**Created on {} **""".format(
 207                time.strftime("%m/%d/%Y, %H:%M:%S", time.localtime()))
 208
 209     def generate_cloud_init(self):
 210         """
 211             **Creates the cloud-init content**
 212
 213             :returns: cloud-init content
 214         """
 215         return "---\n#cloud-config"
 216
 217     def create_files(self, files, file_content, package_type):
 218         """
 219             **Creates the files given the file list and type**
 220
 221             :params:
 222                 - files: is the list of files structure
 223                 - file_content: is the content of the descriptor rendered by the template
 224                 - package_type: is the type of package to filter the creation structure
 225
 226             :return: None
 227         """
 228         self._logger.debug("")
 229         for file_item, file_package, file_type in files:
 230             if package_type == file_package:
 231                 if file_type == "descriptor":
 232                     self.save_file(file_item, file_content)
 233                 elif file_type == "readme":
 234                     self.save_file(file_item, self.generate_readme())
 235                 elif file_type == "cloud_init":
 236                     self.save_file(file_item, self.generate_cloud_init())
 237
 238     def check_files_folders(self, path_list, override):
 239         """
 240             **Find files and folders missing given a directory structure {"folders": [], "files": []}**
 241
 242             :params:
 243                 - path_list: is the list of files and folders to be created
 244                 - override: is the flag used to indicate the creation of the list even if the file exist to override it
 245
 246             :return: Missing paths Dict
 247         """
 248         self._logger.debug("")
 249         missing_paths = {}
 250         folders = []
 251         files = []
 252         for folder in path_list.get("folders"):
 253             if not os.path.exists(folder[0]):
 254                 folders.append(folder)
 255         missing_paths["folders"] = folders
 256
 257         for file_item in path_list.get("files"):
 258             if not os.path.exists(file_item[0]) or override is True:
 259                 files.append(file_item)
 260         missing_paths["files"] = files
 261
 262         return missing_paths
 263
 264     def build_all_charms(self, package_folder, skip_charm_build):
 265         """
 266             **Read the descriptor file, check that the charms referenced are in the folder and compiles them**
 267
 268             :params:
 269                 - packet_folder: is the location of the package
 270             :return: Files and Folders not found. In case of override, it will return all file list
 271         """
 272         listCharms = []
 273         self._logger.debug("")
 274         descriptor_file = False
 275         descriptors_paths = [f for f in glob.glob(package_folder + "/*.yaml")]
 276         for file in descriptors_paths:
 277             if 'nfd.yaml' in file:
 278                 descriptor_file = True
 279                 listCharms = self.charms_search(file, 'vnf')
 280             if 'nsd.yaml' in file:
 281                 descriptor_file = True
 282                 listCharms = self.charms_search(file, 'ns')
 283         if not descriptor_file:
 284             raise ClientException ('descriptor name is not correct in: {}'.format(package_folder))
 285         if listCharms and not skip_charm_build:
 286             for charmName in listCharms:
 287                 if os.path.isdir('{}/charms/layers/{}'.format(package_folder,charmName)):
 288                     self.charm_build(package_folder, charmName)
 289                 else:
 290                     if not os.path.isdir('{}/charms/{}'.format(package_folder,charmName)):
 291                         raise ClientException ('The charm: {} referenced in the descriptor file '
 292                                                'is not present either in {}/charms or in {}/charms/layers'.
 293                                                format(charmName, package_folder,package_folder))
 294         return listCharms
 295
 296     def discover_folder_structure(self, base_directory, name, override):
 297         """
 298             **Discover files and folders structure for OSM descriptors given a base_directory and name**
 299
 300             :params:
 301                 - base_directory: is the location of the package to be created
 302                 - name: is the name of the package
 303                 - override: is the flag used to indicate the creation of the list even if the file exist to override it
 304             :return: Files and Folders not found. In case of override, it will return all file list
 305         """
 306         self._logger.debug("")
 307         prefix = "{}/{}".format(base_directory, name)
 308         files_folders = {"folders": [("{}_ns".format(prefix), "ns"),
 309                                      ("{}_ns/icons".format(prefix), "ns"),
 310                                      ("{}_ns/charms".format(prefix), "ns"),
 311                                      ("{}_vnf".format(name), "vnf"),
 312                                      ("{}_vnf/charms".format(prefix), "vnf"),
 313                                      ("{}_vnf/cloud_init".format(prefix), "vnf"),
 314                                      ("{}_vnf/images".format(prefix), "vnf"),
 315                                      ("{}_vnf/icons".format(prefix), "vnf"),
 316                                      ("{}_vnf/scripts".format(prefix), "vnf"),
 317                                      ("{}_nst".format(prefix), "nst"),
 318                                      ("{}_nst/icons".format(prefix), "nst")
 319                                      ],
 320                          "files": [("{}_ns/{}_nsd.yaml".format(prefix, name), "ns", "descriptor"),
 321                                    ("{}_ns/README.md".format(prefix), "ns", "readme"),
 322                                    ("{}_vnf/{}_vnfd.yaml".format(prefix, name), "vnf", "descriptor"),
 323                                    ("{}_vnf/cloud_init/cloud-config.txt".format(prefix), "vnf", "cloud_init"),
 324                                    ("{}_vnf/README.md".format(prefix), "vnf", "readme"),
 325                                    ("{}_nst/{}_nst.yaml".format(prefix, name), "nst", "descriptor"),
 326                                    ("{}_nst/README.md".format(prefix), "nst", "readme")
 327                                    ]
 328                          }
 329         missing_files_folders = self.check_files_folders(files_folders, override)
 330         # print("Missing files and folders: {}".format(missing_files_folders))
 331         return missing_files_folders
 332
 333     def charm_build(self, charms_folder, build_name):
 334         """
 335         Build the charms inside the package.
 336         params: package_folder is the name of the folder where is the charms to compile.
 337                 build_name is the name of the layer or interface
 338         """
 339         self._logger.debug("")
 340         os.environ['JUJU_REPOSITORY'] = "{}/charms".format(charms_folder)
 341         os.environ['CHARM_LAYERS_DIR'] = "{}/layers".format(os.environ['JUJU_REPOSITORY'])
 342         os.environ['CHARM_INTERFACES_DIR'] = "{}/interfaces".format(os.environ['JUJU_REPOSITORY'])
 343         os.environ['CHARM_BUILD_DIR'] = "{}/charms/builds".format(charms_folder)
 344         src_folder = '{}/{}'.format(os.environ['CHARM_LAYERS_DIR'], build_name)
 345         result = subprocess.run(["charm", "build", "{}".format(src_folder)])
 346         if result.returncode == 1:
 347             raise ClientException("failed to build the charm: {}".format(src_folder))
 348         self._logger.verbose("charm: {} compiled".format(src_folder))
 349
 350     def build_tarfile(self, package_folder, charm_list=None):
 351         """
 352         Creates a .tar.gz file given a package_folder
 353         params: package_folder is the name of the folder to be packaged
 354         returns: .tar.gz name
 355         """
 356         self._logger.debug("")
 357         ignore_patterns = "'*layers*', '*interfaces*'"
 358         try:
 359             directory_name = self.create_temp_dir(package_folder, ignore_patterns, charm_list)
 360             cwd = os.getcwd()
 361             os.chdir(directory_name)
 362             self.calculate_checksum(package_folder)
 363             with tarfile.open("{}.tar.gz".format(package_folder), mode='w:gz') as archive:
 364                 print("Adding File: {}".format(package_folder))
 365                 archive.add('{}'.format(package_folder), recursive=True)
 366             #return "Created {}.tar.gz".format(package_folder)
 367             #self.build("{}".format(os.path.basename(package_folder)))
 368             os.chdir(cwd)
 369         except:
 370             shutil.rmtree(os.path.join(package_folder, "tmp"))
 371             raise ClientException('failure to manipulate the result of the compilation')
 372         os.rename("{}/{}.tar.gz".format(directory_name, os.path.basename(package_folder)),
 373                   "{}.tar.gz".format(os.path.basename(package_folder)))
 374         os.rename("{}/{}/checksums.txt".format(directory_name, os.path.basename(package_folder)),
 375                   "{}/checksums.txt".format(package_folder))
 376         shutil.rmtree(os.path.join(package_folder, "tmp"))
 377         self._logger.verbose("package created: {}.tar.gz".format(os.path.basename(package_folder)))
 378         return "{}.tar.gz".format(package_folder)
 379
 380     def create_temp_dir(self, package_folder, ignore_patterns=None, charm_list=None):
 381         """
 382         Method to create a temporary folder where we can move the files in package_folder, which do not
 383         meet the pattern defined in ignore_patterns
 384         """
 385         self._logger.debug("")
 386         ignore = shutil.ignore_patterns(ignore_patterns)
 387         os.makedirs("{}/tmp".format(package_folder), exist_ok=True)
 388         directory_name = os.path.abspath("{}/tmp".format(package_folder))
 389         os.makedirs("{}/{}".format(directory_name, os.path.basename(package_folder),exist_ok=True))
 390         for item in os.listdir(package_folder):
 391             if item != "tmp":
 392                 s = os.path.join(package_folder, item)
 393                 d = os.path.join(os.path.join(directory_name, os.path.basename(package_folder)), item)
 394                 if os.path.isdir(s):
 395                     if item == "charms":
 396                         s = os.path.join(s, "builds")
 397                         if not os.path.exists(s):
 398                             os.makedirs(s)
 399                         for i in os.listdir(s):
 400                             if i in charm_list:
 401                                 s_charm = os.path.join(s, i)
 402                                 # d_charm = os.path.join(package_folder, item, i)
 403                                 d_temp = os.path.join(d, i)
 404                                 # if os.path.exists(d_charm):
 405                                 #     shutil.rmtree(d_charm)
 406                                 shutil.copytree(s_charm, d_temp, symlinks = True, ignore = ignore)
 407                                 # shutil.copytree(s_charm, d_charm, symlinks = True, ignore = ignore)
 408                     else:
 409                         shutil.copytree(s, d, symlinks = True, ignore = ignore)
 410                 else:
 411                     shutil.copy2(s, d)
 412         return directory_name
 413
 414     def charms_search(self, descriptor_file, desc_type):
 415         self._logger.debug("")
 416         dict = {}
 417         list = []
 418         with open("{}".format(descriptor_file)) as yaml_desc:
 419             dict = yaml.safe_load(yaml_desc)
 420             for k1, v1 in dict.items():
 421                 for k2, v2 in v1.items():
 422                     for entry in v2:
 423                         if '{}-configuration'.format(desc_type) in entry:
 424                             name = entry['{}-configuration'.format(desc_type)]
 425                             for k3, v3 in name.items():
 426                                 if 'charm' in v3:
 427                                     list.append((v3['charm']))
 428                         if 'vdu' in entry:
 429                             name = entry['vdu']
 430                             for vdu in name:
 431                                 if 'vdu-configuration' in vdu:
 432                                     for k4, v4 in vdu['vdu-configuration'].items():
 433                                         if 'charm' in v4:
 434                                             list.append((v4['charm']))
 435         return list
 436