osmclient/common/package_tool.py

   1 # /bin/env python3
   2 # Copyright 2019 ATOS
   3 #
   4 # All Rights Reserved.
   5 #
   6 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 #    not use this file except in compliance with the License. You may obtain
   8 #    a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #    Unless required by applicable law or agreed to in writing, software
  13 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 #    License for the specific language governing permissions and limitations
  16 #    under the License.
  17
  18 from osmclient.common.exceptions import ClientException
  19 import os
  20 import glob
  21 import time
  22 import tarfile
  23 import hashlib
  24 from osm_im.validation import Validation as validation_im
  25 from jinja2 import Environment, PackageLoader
  26 import subprocess
  27 import shutil
  28 import yaml
  29 import logging
  30 import pathlib
  31
  32
  33 class PackageTool(object):
  34     def __init__(self, client=None):
  35         self._client = client
  36         self._logger = logging.getLogger('osmclient')
  37
  38     def create(self, package_type, base_directory, package_name, override, image, vdus, vcpu, memory, storage,
  39                interfaces, vendor, detailed, netslice_subnets, netslice_vlds):
  40         """
  41             **Create a package descriptor**
  42
  43             :params:
  44                 - package_type: [vnf, ns, nst]
  45                 - base directory: path of destination folder
  46                 - package_name: is the name of the package to be created
  47                 - image: specify the image of the vdu
  48                 - vcpu: number of virtual cpus of the vdu
  49                 - memory: amount of memory in MB pf the vdu
  50                 - storage: amount of storage in GB of the vdu
  51                 - interfaces: number of interfaces besides management interface
  52                 - vendor: vendor name of the vnf/ns
  53                 - detailed: include all possible values for NSD, VNFD, NST
  54                 - netslice_subnets: number of netslice_subnets for the NST
  55                 - netslice_vlds: number of virtual link descriptors for the NST
  56
  57             :return: status
  58         """
  59         self._logger.debug("")
  60         # print("location: {}".format(osmclient.__path__))
  61         file_loader = PackageLoader("osmclient")
  62         env = Environment(loader=file_loader)
  63         if package_type == 'ns':
  64             template = env.get_template('nsd.yaml.j2')
  65             content = {"name": package_name, "vendor": vendor, "vdus": vdus, "clean": False, "interfaces": interfaces,
  66                        "detailed": detailed}
  67         elif package_type == 'vnf':
  68             template = env.get_template('vnfd.yaml.j2')
  69             content = {"name": package_name, "vendor": vendor, "vdus": vdus, "clean": False, "interfaces": interfaces,
  70                        "image": image, "vcpu": vcpu, "memory": memory, "storage": storage, "detailed": detailed}
  71         elif package_type == 'nst':
  72             template = env.get_template('nst.yaml.j2')
  73             content = {"name": package_name, "vendor": vendor, "interfaces": interfaces,
  74                        "netslice_subnets": netslice_subnets, "netslice_vlds": netslice_vlds, "detailed": detailed}
  75         else:
  76             raise ClientException("Wrong descriptor type {}. Options: ns, vnf, nst".format(package_type))
  77
  78         # print("To be rendered: {}".format(content))
  79         output = template.render(content)
  80         # print(output)
  81
  82         structure = self.discover_folder_structure(base_directory, package_name, override)
  83         if structure.get("folders"):
  84             self.create_folders(structure["folders"], package_type)
  85         if structure.get("files"):
  86             self.create_files(structure["files"], output, package_type)
  87         return "Created"
  88
  89     def validate(self, base_directory, recursive=True):
  90         """
  91             **Validate OSM Descriptors given a path**
  92
  93             :params:
  94                 - base_directory is the root path for all descriptors
  95
  96             :return: List of dict of validated descriptors. keys: type, path, valid, error
  97         """
  98         self._logger.debug("")
  99         table = []
 100         if recursive:
 101             descriptors_paths = [f for f in glob.glob(base_directory + "/**/*.yaml", recursive=recursive)]
 102         else:
 103             descriptors_paths = [f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)]
 104         print("Base directory: {}".format(base_directory))
 105         print("{} Descriptors found to validate".format(len(descriptors_paths)))
 106         for desc_path in descriptors_paths:
 107             with open(desc_path) as descriptor_file:
 108                 descriptor_data = descriptor_file.read()
 109             desc_type = "-"
 110             try:
 111                 desc_type, descriptor_data = validation_im.yaml_validation(self, descriptor_data)
 112                 validation_im.pyangbind_validation(self, desc_type, descriptor_data)
 113                 table.append({"type": desc_type, "path": desc_path, "valid": "OK", "error": "-"})
 114             except Exception as e:
 115                 table.append({"type": desc_type, "path": desc_path, "valid": "ERROR", "error": str(e)})
 116         return table
 117
 118     def build(self, package_folder, skip_validation=False, skip_charm_build=False):
 119         """
 120             **Creates a .tar.gz file given a package_folder**
 121
 122             :params:
 123                 - package_folder: is the name of the folder to be packaged
 124                 - skip_validation: is the flag to validate or not the descriptors on the folder before build
 125
 126             :returns: message result for the build process
 127         """
 128         self._logger.debug("")
 129         package_folder = package_folder.rstrip('/')
 130         if not os.path.exists("{}".format(package_folder)):
 131             return "Fail, package is not in the specified path"
 132         if not skip_validation:
 133             print('Validating package {}'.format(package_folder))
 134             results = self.validate(package_folder, recursive=False)
 135             if results:
 136                 for result in results:
 137                     if result["valid"] != "OK":
 138                         raise ClientException("There was an error validating the file {} with error: {}"
 139                                               .format(result["path"], result["error"]))
 140                 print('Validation OK')
 141             else:
 142                 raise ClientException("No descriptor file found in: {}".format(package_folder))
 143         charm_list = self.build_all_charms(package_folder, skip_charm_build)
 144         return self.build_tarfile(package_folder, charm_list)
 145
 146     def calculate_checksum(self, package_folder):
 147         """
 148             **Function to calculate the checksum given a folder**
 149
 150             :params:
 151                 - package_folder: is the folder where we have the files to calculate the checksum
 152             :returns: None
 153         """
 154         self._logger.debug("")
 155         files = [f for f in glob.glob(package_folder + "/**/*.*", recursive=True) if os.path.isfile(f)]
 156         with open("{}/checksums.txt".format(package_folder), "w+") as checksum:
 157             for file_item in files:
 158                 if "checksums.txt" in file_item:
 159                     continue
 160                 # from https://www.quickprogrammingtips.com/python/how-to-calculate-md5-hash-of-a-file-in-python.html
 161                 md5_hash = hashlib.md5()
 162                 with open(file_item, "rb") as f:
 163                     # Read and update hash in chunks of 4K
 164                     for byte_block in iter(lambda: f.read(4096), b""):
 165                         md5_hash.update(byte_block)
 166                     checksum.write("{}\t{}\n".format(md5_hash.hexdigest(), file_item))
 167
 168     def create_folders(self, folders, package_type):
 169         """
 170             **Create folder given a list of folders**
 171
 172             :params:
 173                 - folders: [List] list of folders paths to be created
 174                 - package_type: is the type of package to be created
 175             :return: None
 176         """
 177
 178         for folder in folders:
 179             try:
 180                 # print("Folder {} == package_type {}".format(folder[1], package_type))
 181                 if folder[1] == package_type:
 182                     print("Creating folder:\t{}".format(folder[0]))
 183                     os.makedirs(folder[0])
 184             except FileExistsError:
 185                 pass
 186
 187     def save_file(self, file_name, file_body):
 188         """
 189             **Create a file given a name and the content**
 190
 191             :params:
 192                 - file_name: is the name of the file with the relative route
 193                 - file_body: is the content of the file
 194             :return: None
 195         """
 196         print("Creating file:  \t{}".format(file_name))
 197         try:
 198             with open(file_name, "w+") as f:
 199                 f.write(file_body)
 200         except Exception as e:
 201             raise ClientException(e)
 202
 203     def generate_readme(self):
 204         """
 205             **Creates the README content**
 206
 207             :returns: readme content
 208         """
 209         return """# Descriptor created by OSM descriptor package generated\n\n**Created on {} **""".format(
 210                time.strftime("%m/%d/%Y, %H:%M:%S", time.localtime()))
 211
 212     def generate_cloud_init(self):
 213         """
 214             **Creates the cloud-init content**
 215
 216             :returns: cloud-init content
 217         """
 218         return "---\n#cloud-config"
 219
 220     def create_files(self, files, file_content, package_type):
 221         """
 222             **Creates the files given the file list and type**
 223
 224             :params:
 225                 - files: is the list of files structure
 226                 - file_content: is the content of the descriptor rendered by the template
 227                 - package_type: is the type of package to filter the creation structure
 228
 229             :return: None
 230         """
 231         self._logger.debug("")
 232         for file_item, file_package, file_type in files:
 233             if package_type == file_package:
 234                 if file_type == "descriptor":
 235                     self.save_file(file_item, file_content)
 236                 elif file_type == "readme":
 237                     self.save_file(file_item, self.generate_readme())
 238                 elif file_type == "cloud_init":
 239                     self.save_file(file_item, self.generate_cloud_init())
 240
 241     def check_files_folders(self, path_list, override):
 242         """
 243             **Find files and folders missing given a directory structure {"folders": [], "files": []}**
 244
 245             :params:
 246                 - path_list: is the list of files and folders to be created
 247                 - override: is the flag used to indicate the creation of the list even if the file exist to override it
 248
 249             :return: Missing paths Dict
 250         """
 251         self._logger.debug("")
 252         missing_paths = {}
 253         folders = []
 254         files = []
 255         for folder in path_list.get("folders"):
 256             if not os.path.exists(folder[0]):
 257                 folders.append(folder)
 258         missing_paths["folders"] = folders
 259
 260         for file_item in path_list.get("files"):
 261             if not os.path.exists(file_item[0]) or override is True:
 262                 files.append(file_item)
 263         missing_paths["files"] = files
 264
 265         return missing_paths
 266
 267     def build_all_charms(self, package_folder, skip_charm_build):
 268         """
 269             **Read the descriptor file, check that the charms referenced are in the folder and compiles them**
 270
 271             :params:
 272                 - packet_folder: is the location of the package
 273             :return: Files and Folders not found. In case of override, it will return all file list
 274         """
 275         self._logger.debug("")
 276         listCharms = []
 277         descriptor_file = False
 278         descriptors_paths = [f for f in glob.glob(package_folder + "/*.yaml")]
 279         for file in descriptors_paths:
 280             if file.endswith('nfd.yaml'):
 281                 descriptor_file = True
 282                 listCharms = self.charms_search(file, 'vnf')
 283             if file.endswith('nsd.yaml'):
 284                 descriptor_file = True
 285                 listCharms = self.charms_search(file, 'ns')
 286         print("List of charms in the descriptor: {}".format(listCharms))
 287         if not descriptor_file:
 288             raise ClientException ('descriptor name is not correct in: {}'.format(package_folder))
 289         if listCharms and not skip_charm_build:
 290             for charmName in listCharms:
 291                 if os.path.isdir('{}/charms/layers/{}'.format(package_folder,charmName)):
 292                     print('Building charm {}/charms/layers/{}'.format(package_folder, charmName))
 293                     self.charm_build(package_folder, charmName)
 294                     print('Charm built: {}'.format(charmName))
 295                 else:
 296                     if not os.path.isdir('{}/charms/{}'.format(package_folder,charmName)):
 297                         raise ClientException ('The charm: {} referenced in the descriptor file '
 298                                                'is not present either in {}/charms or in {}/charms/layers'.
 299                                                format(charmName, package_folder,package_folder))
 300         self._logger.debug("Return list of charms: {}".format(listCharms))
 301         return listCharms
 302
 303     def discover_folder_structure(self, base_directory, name, override):
 304         """
 305             **Discover files and folders structure for OSM descriptors given a base_directory and name**
 306
 307             :params:
 308                 - base_directory: is the location of the package to be created
 309                 - name: is the name of the package
 310                 - override: is the flag used to indicate the creation of the list even if the file exist to override it
 311             :return: Files and Folders not found. In case of override, it will return all file list
 312         """
 313         self._logger.debug("")
 314         prefix = "{}/{}".format(base_directory, name)
 315         files_folders = {"folders": [("{}_ns".format(prefix), "ns"),
 316                                      ("{}_ns/icons".format(prefix), "ns"),
 317                                      ("{}_ns/charms".format(prefix), "ns"),
 318                                      ("{}_vnf".format(name), "vnf"),
 319                                      ("{}_vnf/charms".format(prefix), "vnf"),
 320                                      ("{}_vnf/cloud_init".format(prefix), "vnf"),
 321                                      ("{}_vnf/images".format(prefix), "vnf"),
 322                                      ("{}_vnf/icons".format(prefix), "vnf"),
 323                                      ("{}_vnf/scripts".format(prefix), "vnf"),
 324                                      ("{}_nst".format(prefix), "nst"),
 325                                      ("{}_nst/icons".format(prefix), "nst")
 326                                      ],
 327                          "files": [("{}_ns/{}_nsd.yaml".format(prefix, name), "ns", "descriptor"),
 328                                    ("{}_ns/README.md".format(prefix), "ns", "readme"),
 329                                    ("{}_vnf/{}_vnfd.yaml".format(prefix, name), "vnf", "descriptor"),
 330                                    ("{}_vnf/cloud_init/cloud-config.txt".format(prefix), "vnf", "cloud_init"),
 331                                    ("{}_vnf/README.md".format(prefix), "vnf", "readme"),
 332                                    ("{}_nst/{}_nst.yaml".format(prefix, name), "nst", "descriptor"),
 333                                    ("{}_nst/README.md".format(prefix), "nst", "readme")
 334                                    ]
 335                          }
 336         missing_files_folders = self.check_files_folders(files_folders, override)
 337         # print("Missing files and folders: {}".format(missing_files_folders))
 338         return missing_files_folders
 339
 340     def charm_build(self, charms_folder, build_name):
 341         """
 342         Build the charms inside the package.
 343         params: package_folder is the name of the folder where is the charms to compile.
 344                 build_name is the name of the layer or interface
 345         """
 346         self._logger.debug("")
 347         os.environ['JUJU_REPOSITORY'] = "{}/charms".format(charms_folder)
 348         os.environ['CHARM_LAYERS_DIR'] = "{}/layers".format(os.environ['JUJU_REPOSITORY'])
 349         os.environ['CHARM_INTERFACES_DIR'] = "{}/interfaces".format(os.environ['JUJU_REPOSITORY'])
 350         os.environ['CHARM_BUILD_DIR'] = "{}/charms/builds".format(charms_folder)
 351         if not os.path.exists(os.environ['CHARM_BUILD_DIR']):
 352             os.makedirs(os.environ['CHARM_BUILD_DIR'])
 353         src_folder = '{}/{}'.format(os.environ['CHARM_LAYERS_DIR'], build_name)
 354         result = subprocess.run(["charm", "build", "{}".format(src_folder)])
 355         if result.returncode == 1:
 356             raise ClientException("failed to build the charm: {}".format(src_folder))
 357         self._logger.verbose("charm {} built".format(src_folder))
 358
 359     def build_tarfile(self, package_folder, charm_list=None):
 360         """
 361         Creates a .tar.gz file given a package_folder
 362         params: package_folder is the name of the folder to be packaged
 363         returns: .tar.gz name
 364         """
 365         self._logger.debug("")
 366         cwd = None
 367         try:
 368             directory_name, package_name = self.create_temp_dir(package_folder, charm_list)
 369             cwd = os.getcwd()
 370             os.chdir(directory_name)
 371             self.calculate_checksum(package_name)
 372             with tarfile.open("{}.tar.gz".format(package_name), mode='w:gz') as archive:
 373                 print("Adding File: {}".format(package_name))
 374                 archive.add('{}'.format(package_name), recursive=True)
 375             # return "Created {}.tar.gz".format(package_folder)
 376             # self.build("{}".format(os.path.basename(package_folder)))
 377             os.chdir(cwd)
 378             cwd = None
 379             created_package = "{}/{}.tar.gz".format(os.path.dirname(package_folder) or '.', package_name)
 380             os.rename("{}/{}.tar.gz".format(directory_name, package_name),
 381                       created_package)
 382             os.rename("{}/{}/checksums.txt".format(directory_name, package_name),
 383                       "{}/checksums.txt".format(package_folder))
 384             print("Package created: {}".format(created_package))
 385             return created_package
 386         except Exception as exc:
 387             raise ClientException('failure during build of targz file (create temp dir, calculate checksum, '
 388                                   'tar.gz file): {}'.format(exc))
 389         finally:
 390             if cwd:
 391                 os.chdir(cwd)
 392             shutil.rmtree(os.path.join(package_folder, "tmp"))
 393
 394     def create_temp_dir(self, package_folder, charm_list=None):
 395         """
 396         Method to create a temporary folder where we can move the files in package_folder
 397         """
 398         self._logger.debug("")
 399         ignore_patterns = ('.gitignore')
 400         ignore = shutil.ignore_patterns(ignore_patterns)
 401         directory_name = os.path.abspath(package_folder)
 402         package_name = os.path.basename(directory_name)
 403         directory_name += "/tmp"
 404         os.makedirs("{}/{}".format(directory_name, package_name), exist_ok=True)
 405         self._logger.debug("Makedirs DONE: {}/{}".format(directory_name, package_name))
 406         for item in os.listdir(package_folder):
 407             self._logger.debug("Item: {}".format(item))
 408             if item != "tmp":
 409                 s = os.path.join(package_folder, item)
 410                 d = os.path.join(os.path.join(directory_name, package_name), item)
 411                 if os.path.isdir(s):
 412                     if item == "charms":
 413                         os.makedirs(d, exist_ok=True)
 414                         s_builds = os.path.join(s, "builds")
 415                         for charm in charm_list:
 416                             self._logger.debug("Copying charm {}".format(charm))
 417                             if charm in os.listdir(s):
 418                                 s_charm = os.path.join(s, charm)
 419                             elif charm in os.listdir(s_builds):
 420                                 s_charm = os.path.join(s_builds, charm)
 421                             else:
 422                                 raise ClientException('The charm {} referenced in the descriptor file '
 423                                                       'could not be found in {}/charms or in {}/charms/builds'.
 424                                                       format(charm, package_folder, package_folder))
 425                             d_temp = os.path.join(d, charm)
 426                             self._logger.debug("Copying tree: {} -> {}".format(s_charm, d_temp))
 427                             shutil.copytree(s_charm, d_temp, symlinks=True, ignore=ignore)
 428                             self._logger.debug("DONE")
 429                     else:
 430                         self._logger.debug("Copying tree: {} -> {}".format(s,d))
 431                         shutil.copytree(s, d, symlinks=True, ignore=ignore)
 432                         self._logger.debug("DONE")
 433                 else:
 434                     if item in ignore_patterns:
 435                         continue
 436                     self._logger.debug("Copying file: {} -> {}".format(s,d))
 437                     shutil.copy2(s, d)
 438                     self._logger.debug("DONE")
 439         return directory_name, package_name
 440
 441     def charms_search(self, descriptor_file, desc_type):
 442         self._logger.debug("")
 443
 444         descriptor_dict = {}
 445         charm_list = []
 446         bundle_list = []
 447
 448         with open("{}".format(descriptor_file)) as yaml_desc:
 449             descriptor_dict = yaml.safe_load(yaml_desc)
 450             for _, v1 in descriptor_dict.items():
 451                 for _, v2 in v1.items():
 452                     for entry in v2:
 453                         if "{}-configuration".format(desc_type) in entry:
 454                             name = entry["{}-configuration".format(desc_type)]
 455                             for _, v3 in name.items():
 456                                 if "charm" in v3:
 457                                     charm_list.append((v3["charm"]))
 458                         if "vdu" in entry:
 459                             name = entry["vdu"]
 460                             for vdu in name:
 461                                 if "vdu-configuration" in vdu:
 462                                     for _, v4 in vdu["vdu-configuration"].items():
 463                                         if "charm" in v4:
 464                                             charm_list.append((v4["charm"]))
 465                         if "kdu" in entry:
 466                             name = entry["kdu"]
 467                             for kdu in name:
 468                                 if "juju-bundle" in kdu:
 469                                     bundle_list.append(kdu["juju-bundle"])
 470
 471         path = pathlib.Path(descriptor_file).parent
 472         for bundle in bundle_list:
 473             descriptor_dict = {}
 474             with open("{}/juju-bundles/{}".format(path, bundle)) as yaml_desc:
 475                 descriptor_dict = yaml.safe_load(yaml_desc)
 476                 if "applications" in descriptor_dict:
 477                     for _, v in descriptor_dict["applications"].items():
 478                         charm_list.append(pathlib.Path(v["charm"]).name)
 479
 480         return charm_list
 481