osmclient/common/package_tool.py

   1 # /bin/env python3
   2 # Copyright 2019 ATOS
   3 #
   4 # All Rights Reserved.
   5 #
   6 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 #    not use this file except in compliance with the License. You may obtain
   8 #    a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #    Unless required by applicable law or agreed to in writing, software
  13 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 #    License for the specific language governing permissions and limitations
  16 #    under the License.
  17
  18 from osmclient.common.exceptions import ClientException
  19 import os
  20 import glob
  21 import time
  22 import tarfile
  23 import hashlib
  24 from osm_im.validation import Validation as validation_im
  25 from jinja2 import Environment, PackageLoader
  26 import subprocess
  27 import shutil
  28 import yaml
  29 import logging
  30
  31 class PackageTool(object):
  32     def __init__(self, client=None):
  33         self._client = client
  34         self._logger = logging.getLogger('osmclient')
  35
  36     def create(self, package_type, base_directory, package_name, override, image, vdus, vcpu, memory, storage,
  37                interfaces, vendor, detailed, netslice_subnets, netslice_vlds):
  38         """
  39             **Create a package descriptor**
  40
  41             :params:
  42                 - package_type: [vnf, ns, nst]
  43                 - base directory: path of destination folder
  44                 - package_name: is the name of the package to be created
  45                 - image: specify the image of the vdu
  46                 - vcpu: number of virtual cpus of the vdu
  47                 - memory: amount of memory in MB pf the vdu
  48                 - storage: amount of storage in GB of the vdu
  49                 - interfaces: number of interfaces besides management interface
  50                 - vendor: vendor name of the vnf/ns
  51                 - detailed: include all possible values for NSD, VNFD, NST
  52                 - netslice_subnets: number of netslice_subnets for the NST
  53                 - netslice_vlds: number of virtual link descriptors for the NST
  54
  55             :return: status
  56         """
  57         self._logger.debug("")
  58         # print("location: {}".format(osmclient.__path__))
  59         file_loader = PackageLoader("osmclient")
  60         env = Environment(loader=file_loader)
  61         if package_type == 'ns':
  62             template = env.get_template('nsd.yaml.j2')
  63             content = {"name": package_name, "vendor": vendor, "vdus": vdus, "clean": False, "interfaces": interfaces,
  64                        "detailed": detailed}
  65         elif package_type == 'vnf':
  66             template = env.get_template('vnfd.yaml.j2')
  67             content = {"name": package_name, "vendor": vendor, "vdus": vdus, "clean": False, "interfaces": interfaces,
  68                        "image": image, "vcpu": vcpu, "memory": memory, "storage": storage, "detailed": detailed}
  69         elif package_type == 'nst':
  70             template = env.get_template('nst.yaml.j2')
  71             content = {"name": package_name, "vendor": vendor, "interfaces": interfaces,
  72                        "netslice_subnets": netslice_subnets, "netslice_vlds": netslice_vlds, "detailed": detailed}
  73         else:
  74             raise ClientException("Wrong descriptor type {}. Options: ns, vnf, nst".format(package_type))
  75
  76         # print("To be rendered: {}".format(content))
  77         output = template.render(content)
  78         # print(output)
  79
  80         structure = self.discover_folder_structure(base_directory, package_name, override)
  81         if structure.get("folders"):
  82             self.create_folders(structure["folders"], package_type)
  83         if structure.get("files"):
  84             self.create_files(structure["files"], output, package_type)
  85         return "Created"
  86
  87     def validate(self, base_directory, recursive=True):
  88         """
  89             **Validate OSM Descriptors given a path**
  90
  91             :params:
  92                 - base_directory is the root path for all descriptors
  93
  94             :return: List of dict of validated descriptors. keys: type, path, valid, error
  95         """
  96         self._logger.debug("")
  97         table = []
  98         if recursive:
  99             descriptors_paths = [f for f in glob.glob(base_directory + "/**/*.yaml", recursive=recursive)]
 100         else:
 101             descriptors_paths = [f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)]
 102         print("Base directory: {}".format(base_directory))
 103         print("{} Descriptors found to validate".format(len(descriptors_paths)))
 104         for desc_path in descriptors_paths:
 105             with open(desc_path) as descriptor_file:
 106                 descriptor_data = descriptor_file.read()
 107             desc_type = "-"
 108             try:
 109                 desc_type, descriptor_data = validation_im.yaml_validation(self, descriptor_data)
 110                 validation_im.pyangbind_validation(self, desc_type, descriptor_data)
 111                 table.append({"type": desc_type, "path": desc_path, "valid": "OK", "error": "-"})
 112             except Exception as e:
 113                 table.append({"type": desc_type, "path": desc_path, "valid": "ERROR", "error": str(e)})
 114         return table
 115
 116     def build(self, package_folder, skip_validation=False, skip_charm_build=False):
 117         """
 118             **Creates a .tar.gz file given a package_folder**
 119
 120             :params:
 121                 - package_folder: is the name of the folder to be packaged
 122                 - skip_validation: is the flag to validate or not the descriptors on the folder before build
 123
 124             :returns: message result for the build process
 125         """
 126         self._logger.debug("")
 127         package_folder = package_folder.rstrip('/')
 128         if not os.path.exists("{}".format(package_folder)):
 129             return "Fail, package is not in the specified path"
 130         if not skip_validation:
 131             print('Validating package {}'.format(package_folder))
 132             results = self.validate(package_folder, recursive=False)
 133             if results:
 134                 for result in results:
 135                     if result["valid"] != "OK":
 136                         raise ClientException("There was an error validating the file {} with error: {}"
 137                                               .format(result["path"], result["error"]))
 138                 print('Validation OK')
 139             else:
 140                 raise ClientException("No descriptor file found in: {}".format(package_folder))
 141         charm_list = self.build_all_charms(package_folder, skip_charm_build)
 142         return self.build_tarfile(package_folder, charm_list)
 143
 144     def calculate_checksum(self, package_folder):
 145         """
 146             **Function to calculate the checksum given a folder**
 147
 148             :params:
 149                 - package_folder: is the folder where we have the files to calculate the checksum
 150             :returns: None
 151         """
 152         self._logger.debug("")
 153         files = [f for f in glob.glob(package_folder + "/**/*.*", recursive=True)]
 154         with open("{}/checksums.txt".format(package_folder), "w+") as checksum:
 155             for file_item in files:
 156                 if "checksums.txt" in file_item:
 157                     continue
 158                 # from https://www.quickprogrammingtips.com/python/how-to-calculate-md5-hash-of-a-file-in-python.html
 159                 md5_hash = hashlib.md5()
 160                 with open(file_item, "rb") as f:
 161                     # Read and update hash in chunks of 4K
 162                     for byte_block in iter(lambda: f.read(4096), b""):
 163                         md5_hash.update(byte_block)
 164                     checksum.write("{}\t{}\n".format(md5_hash.hexdigest(), file_item))
 165
 166     def create_folders(self, folders, package_type):
 167         """
 168             **Create folder given a list of folders**
 169
 170             :params:
 171                 - folders: [List] list of folders paths to be created
 172                 - package_type: is the type of package to be created
 173             :return: None
 174         """
 175
 176         for folder in folders:
 177             try:
 178                 # print("Folder {} == package_type {}".format(folder[1], package_type))
 179                 if folder[1] == package_type:
 180                     print("Creating folder:\t{}".format(folder[0]))
 181                     os.makedirs(folder[0])
 182             except FileExistsError:
 183                 pass
 184
 185     def save_file(self, file_name, file_body):
 186         """
 187             **Create a file given a name and the content**
 188
 189             :params:
 190                 - file_name: is the name of the file with the relative route
 191                 - file_body: is the content of the file
 192             :return: None
 193         """
 194         print("Creating file:  \t{}".format(file_name))
 195         try:
 196             with open(file_name, "w+") as f:
 197                 f.write(file_body)
 198         except Exception as e:
 199             raise ClientException(e)
 200
 201     def generate_readme(self):
 202         """
 203             **Creates the README content**
 204
 205             :returns: readme content
 206         """
 207         return """# Descriptor created by OSM descriptor package generated\n\n**Created on {} **""".format(
 208                time.strftime("%m/%d/%Y, %H:%M:%S", time.localtime()))
 209
 210     def generate_cloud_init(self):
 211         """
 212             **Creates the cloud-init content**
 213
 214             :returns: cloud-init content
 215         """
 216         return "---\n#cloud-config"
 217
 218     def create_files(self, files, file_content, package_type):
 219         """
 220             **Creates the files given the file list and type**
 221
 222             :params:
 223                 - files: is the list of files structure
 224                 - file_content: is the content of the descriptor rendered by the template
 225                 - package_type: is the type of package to filter the creation structure
 226
 227             :return: None
 228         """
 229         self._logger.debug("")
 230         for file_item, file_package, file_type in files:
 231             if package_type == file_package:
 232                 if file_type == "descriptor":
 233                     self.save_file(file_item, file_content)
 234                 elif file_type == "readme":
 235                     self.save_file(file_item, self.generate_readme())
 236                 elif file_type == "cloud_init":
 237                     self.save_file(file_item, self.generate_cloud_init())
 238
 239     def check_files_folders(self, path_list, override):
 240         """
 241             **Find files and folders missing given a directory structure {"folders": [], "files": []}**
 242
 243             :params:
 244                 - path_list: is the list of files and folders to be created
 245                 - override: is the flag used to indicate the creation of the list even if the file exist to override it
 246
 247             :return: Missing paths Dict
 248         """
 249         self._logger.debug("")
 250         missing_paths = {}
 251         folders = []
 252         files = []
 253         for folder in path_list.get("folders"):
 254             if not os.path.exists(folder[0]):
 255                 folders.append(folder)
 256         missing_paths["folders"] = folders
 257
 258         for file_item in path_list.get("files"):
 259             if not os.path.exists(file_item[0]) or override is True:
 260                 files.append(file_item)
 261         missing_paths["files"] = files
 262
 263         return missing_paths
 264
 265     def build_all_charms(self, package_folder, skip_charm_build):
 266         """
 267             **Read the descriptor file, check that the charms referenced are in the folder and compiles them**
 268
 269             :params:
 270                 - packet_folder: is the location of the package
 271             :return: Files and Folders not found. In case of override, it will return all file list
 272         """
 273         self._logger.debug("")
 274         listCharms = []
 275         descriptor_file = False
 276         descriptors_paths = [f for f in glob.glob(package_folder + "/*.yaml")]
 277         for file in descriptors_paths:
 278             if file.endswith('nfd.yaml'):
 279                 descriptor_file = True
 280                 listCharms = self.charms_search(file, 'vnf')
 281             if file.endswith('nsd.yaml'):
 282                 descriptor_file = True
 283                 listCharms = self.charms_search(file, 'ns')
 284         print("List of charms in the descriptor: {}".format(listCharms))
 285         if not descriptor_file:
 286             raise ClientException ('descriptor name is not correct in: {}'.format(package_folder))
 287         if listCharms and not skip_charm_build:
 288             for charmName in listCharms:
 289                 if os.path.isdir('{}/charms/layers/{}'.format(package_folder,charmName)):
 290                     print('Building charm {}/charms/layers/{}'.format(package_folder, charmName))
 291                     self.charm_build(package_folder, charmName)
 292                     print('Charm built {}'.format(charmName))
 293                 else:
 294                     if not os.path.isdir('{}/charms/{}'.format(package_folder,charmName)):
 295                         raise ClientException ('The charm: {} referenced in the descriptor file '
 296                                                'is not present either in {}/charms or in {}/charms/layers'.
 297                                                format(charmName, package_folder,package_folder))
 298         self._logger.debug("Return list of charms: {}".format(listCharms))
 299         return listCharms
 300
 301     def discover_folder_structure(self, base_directory, name, override):
 302         """
 303             **Discover files and folders structure for OSM descriptors given a base_directory and name**
 304
 305             :params:
 306                 - base_directory: is the location of the package to be created
 307                 - name: is the name of the package
 308                 - override: is the flag used to indicate the creation of the list even if the file exist to override it
 309             :return: Files and Folders not found. In case of override, it will return all file list
 310         """
 311         self._logger.debug("")
 312         prefix = "{}/{}".format(base_directory, name)
 313         files_folders = {"folders": [("{}_ns".format(prefix), "ns"),
 314                                      ("{}_ns/icons".format(prefix), "ns"),
 315                                      ("{}_ns/charms".format(prefix), "ns"),
 316                                      ("{}_vnf".format(name), "vnf"),
 317                                      ("{}_vnf/charms".format(prefix), "vnf"),
 318                                      ("{}_vnf/cloud_init".format(prefix), "vnf"),
 319                                      ("{}_vnf/images".format(prefix), "vnf"),
 320                                      ("{}_vnf/icons".format(prefix), "vnf"),
 321                                      ("{}_vnf/scripts".format(prefix), "vnf"),
 322                                      ("{}_nst".format(prefix), "nst"),
 323                                      ("{}_nst/icons".format(prefix), "nst")
 324                                      ],
 325                          "files": [("{}_ns/{}_nsd.yaml".format(prefix, name), "ns", "descriptor"),
 326                                    ("{}_ns/README.md".format(prefix), "ns", "readme"),
 327                                    ("{}_vnf/{}_vnfd.yaml".format(prefix, name), "vnf", "descriptor"),
 328                                    ("{}_vnf/cloud_init/cloud-config.txt".format(prefix), "vnf", "cloud_init"),
 329                                    ("{}_vnf/README.md".format(prefix), "vnf", "readme"),
 330                                    ("{}_nst/{}_nst.yaml".format(prefix, name), "nst", "descriptor"),
 331                                    ("{}_nst/README.md".format(prefix), "nst", "readme")
 332                                    ]
 333                          }
 334         missing_files_folders = self.check_files_folders(files_folders, override)
 335         # print("Missing files and folders: {}".format(missing_files_folders))
 336         return missing_files_folders
 337
 338     def charm_build(self, charms_folder, build_name):
 339         """
 340         Build the charms inside the package.
 341         params: package_folder is the name of the folder where is the charms to compile.
 342                 build_name is the name of the layer or interface
 343         """
 344         self._logger.debug("")
 345         os.environ['JUJU_REPOSITORY'] = "{}/charms".format(charms_folder)
 346         os.environ['CHARM_LAYERS_DIR'] = "{}/layers".format(os.environ['JUJU_REPOSITORY'])
 347         os.environ['CHARM_INTERFACES_DIR'] = "{}/interfaces".format(os.environ['JUJU_REPOSITORY'])
 348         os.environ['CHARM_BUILD_DIR'] = "{}/charms/builds".format(charms_folder)
 349         if not os.path.exists(os.environ['CHARM_BUILD_DIR']):
 350             os.makedirs(os.environ['CHARM_BUILD_DIR'])
 351         src_folder = '{}/{}'.format(os.environ['CHARM_LAYERS_DIR'], build_name)
 352         result = subprocess.run(["charm", "build", "{}".format(src_folder)])
 353         if result.returncode == 1:
 354             raise ClientException("failed to build the charm: {}".format(src_folder))
 355         self._logger.verbose("charm {} built".format(src_folder))
 356
 357     def build_tarfile(self, package_folder, charm_list=None):
 358         """
 359         Creates a .tar.gz file given a package_folder
 360         params: package_folder is the name of the folder to be packaged
 361         returns: .tar.gz name
 362         """
 363         self._logger.debug("")
 364         cwd = None
 365         try:
 366             directory_name, package_name = self.create_temp_dir(package_folder, charm_list)
 367             cwd = os.getcwd()
 368             os.chdir(directory_name)
 369             self.calculate_checksum(package_name)
 370             with tarfile.open("{}.tar.gz".format(package_name), mode='w:gz') as archive:
 371                 print("Adding File: {}".format(package_name))
 372                 archive.add('{}'.format(package_name), recursive=True)
 373             #return "Created {}.tar.gz".format(package_folder)
 374             #self.build("{}".format(os.path.basename(package_folder)))
 375             os.chdir(cwd)
 376         except Exception as exc:
 377             if cwd:
 378                 os.chdir(cwd)
 379             shutil.rmtree(os.path.join(package_folder, "tmp"))
 380             raise ClientException('failure during build of targz file (create temp dir, calculate checksum, tar.gz file): {}'.format(exc))
 381         created_package = "{}/{}.tar.gz".format(package_folder, package_name)
 382         os.rename("{}/{}.tar.gz".format(directory_name, package_name),
 383                   created_package)
 384         os.rename("{}/{}/checksums.txt".format(directory_name, package_name),
 385                   "{}/checksums.txt".format(package_folder))
 386         shutil.rmtree(os.path.join(package_folder, "tmp"))
 387         print("Package created: {}".format(created_package))
 388         return created_package
 389
 390     def create_temp_dir(self, package_folder, charm_list=None):
 391         """
 392         Method to create a temporary folder where we can move the files in package_folder
 393         """
 394         self._logger.debug("")
 395         ignore_patterns = ('.gitignore')
 396         ignore = shutil.ignore_patterns(ignore_patterns)
 397         directory_name = os.path.abspath(package_folder)
 398         package_name = os.path.basename(directory_name)
 399         directory_name += "/tmp"
 400         os.makedirs("{}/{}".format(directory_name, package_name), exist_ok=True)
 401         self._logger.debug("Makedirs DONE: {}/{}".format(directory_name, package_name))
 402         for item in os.listdir(package_folder):
 403             self._logger.debug("Item: {}".format(item))
 404             if item != "tmp":
 405                 s = os.path.join(package_folder, item)
 406                 d = os.path.join(os.path.join(directory_name, package_name), item)
 407                 if os.path.isdir(s):
 408                     if item == "charms":
 409                         os.makedirs(d, exist_ok=True)
 410                         s_builds = os.path.join(s, "builds")
 411                         for charm in charm_list:
 412                             self._logger.debug("Copying charm {}".format(charm))
 413                             if charm in os.listdir(s):
 414                                 s_charm = os.path.join(s, charm)
 415                             elif charm in os.listdir(s_builds):
 416                                 s_charm = os.path.join(s_builds, charm)
 417                             else:
 418                                 raise ClientException('The charm {} referenced in the descriptor file '
 419                                                       'could not be found in {}/charms or in {}/charms/builds'.
 420                                                       format(charm, package_folder, package_folder))
 421                             d_temp = os.path.join(d, charm)
 422                             self._logger.debug("Copying tree: {} -> {}".format(s_charm, d_temp))
 423                             shutil.copytree(s_charm, d_temp, symlinks=True, ignore=ignore)
 424                             self._logger.debug("DONE")
 425                     else:
 426                         self._logger.debug("Copying tree: {} -> {}".format(s,d))
 427                         shutil.copytree(s, d, symlinks=True, ignore=ignore)
 428                         self._logger.debug("DONE")
 429                 else:
 430                     if item in ignore_patterns:
 431                         continue
 432                     self._logger.debug("Copying file: {} -> {}".format(s,d))
 433                     shutil.copy2(s, d)
 434                     self._logger.debug("DONE")
 435         return directory_name, package_name
 436
 437     def charms_search(self, descriptor_file, desc_type):
 438         self._logger.debug("")
 439         dict = {}
 440         list = []
 441         with open("{}".format(descriptor_file)) as yaml_desc:
 442             dict = yaml.safe_load(yaml_desc)
 443             for k1, v1 in dict.items():
 444                 for k2, v2 in v1.items():
 445                     for entry in v2:
 446                         if '{}-configuration'.format(desc_type) in entry:
 447                             name = entry['{}-configuration'.format(desc_type)]
 448                             for k3, v3 in name.items():
 449                                 if 'charm' in v3:
 450                                     list.append((v3['charm']))
 451                         if 'vdu' in entry:
 452                             name = entry['vdu']
 453                             for vdu in name:
 454                                 if 'vdu-configuration' in vdu:
 455                                     for k4, v4 in vdu['vdu-configuration'].items():
 456                                         if 'charm' in v4:
 457                                             list.append((v4['charm']))
 458         return list
 459