osmclient/common/package_tool.py

   1 # /bin/env python3
   2 # Copyright 2019 ATOS
   3 #
   4 # All Rights Reserved.
   5 #
   6 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 #    not use this file except in compliance with the License. You may obtain
   8 #    a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #    Unless required by applicable law or agreed to in writing, software
  13 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 #    License for the specific language governing permissions and limitations
  16 #    under the License.
  17
  18 import glob
  19 import hashlib
  20 import logging
  21 import os
  22 import shutil
  23 import subprocess
  24 import tarfile
  25 import time
  26
  27 from jinja2 import Environment, PackageLoader
  28 from osm_im.validation import Validation as validation_im
  29 from osmclient.common.exceptions import ClientException
  30 import yaml
  31
  32
  33 class PackageTool(object):
  34     def __init__(self, client=None):
  35         self._client = client
  36         self._logger = logging.getLogger('osmclient')
  37
  38     def create(self, package_type, base_directory, package_name, override, image, vdus, vcpu, memory, storage,
  39                interfaces, vendor, detailed, netslice_subnets, netslice_vlds):
  40         """
  41             **Create a package descriptor**
  42
  43             :params:
  44                 - package_type: [vnf, ns, nst]
  45                 - base directory: path of destination folder
  46                 - package_name: is the name of the package to be created
  47                 - image: specify the image of the vdu
  48                 - vcpu: number of virtual cpus of the vdu
  49                 - memory: amount of memory in MB pf the vdu
  50                 - storage: amount of storage in GB of the vdu
  51                 - interfaces: number of interfaces besides management interface
  52                 - vendor: vendor name of the vnf/ns
  53                 - detailed: include all possible values for NSD, VNFD, NST
  54                 - netslice_subnets: number of netslice_subnets for the NST
  55                 - netslice_vlds: number of virtual link descriptors for the NST
  56
  57             :return: status
  58         """
  59         self._logger.debug("")
  60         # print("location: {}".format(osmclient.__path__))
  61         file_loader = PackageLoader("osmclient")
  62         env = Environment(loader=file_loader)
  63         if package_type == 'ns':
  64             template = env.get_template('nsd.yaml.j2')
  65             content = {"name": package_name, "vendor": vendor, "vdus": vdus, "clean": False, "interfaces": interfaces,
  66                        "detailed": detailed}
  67         elif package_type == 'vnf':
  68             template = env.get_template('vnfd.yaml.j2')
  69             content = {"name": package_name, "vendor": vendor, "vdus": vdus, "clean": False, "interfaces": interfaces,
  70                        "image": image, "vcpu": vcpu, "memory": memory, "storage": storage, "detailed": detailed}
  71         elif package_type == 'nst':
  72             template = env.get_template('nst.yaml.j2')
  73             content = {"name": package_name, "vendor": vendor, "interfaces": interfaces,
  74                        "netslice_subnets": netslice_subnets, "netslice_vlds": netslice_vlds, "detailed": detailed}
  75         else:
  76             raise ClientException("Wrong descriptor type {}. Options: ns, vnf, nst".format(package_type))
  77
  78         # print("To be rendered: {}".format(content))
  79         output = template.render(content)
  80         # print(output)
  81
  82         structure = self.discover_folder_structure(base_directory, package_name, override)
  83         if structure.get("folders"):
  84             self.create_folders(structure["folders"], package_type)
  85         if structure.get("files"):
  86             self.create_files(structure["files"], output, package_type)
  87         return "Created"
  88
  89     def validate(self, base_directory, recursive=True, old_format=False):
  90         """
  91             **Validate OSM Descriptors given a path**
  92
  93             :params:
  94                 - base_directory is the root path for all descriptors
  95
  96             :return: List of dict of validated descriptors. keys: type, path, valid, error
  97         """
  98         self._logger.debug("")
  99         table = []
 100         if recursive:
 101             descriptors_paths = [f for f in glob.glob(base_directory + "/**/*.yaml", recursive=recursive)]
 102         else:
 103             descriptors_paths = [f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)]
 104         print("Base directory: {}".format(base_directory))
 105         print("{} Descriptors found to validate".format(len(descriptors_paths)))
 106         for desc_path in descriptors_paths:
 107             with open(desc_path) as descriptor_file:
 108                 descriptor_data = descriptor_file.read()
 109             desc_type = "-"
 110             try:
 111                 desc_type, descriptor_data = validation_im.yaml_validation(self, descriptor_data)
 112                 if not old_format:
 113                     if ( desc_type=="vnfd" or desc_type=="nsd" ):
 114                         print("OSM descriptor '{}' written in an unsupported format. Please update to ETSI SOL006 format".format(desc_path))
 115                         print("Package validation skipped. It can still be done with 'osm package-validate --old'")
 116                         print("Package build can still be done with 'osm package-build --skip-validation'")
 117                         raise Exception("Not SOL006 format")
 118                 validation_im.pyangbind_validation(self, desc_type, descriptor_data)
 119                 table.append({"type": desc_type, "path": desc_path, "valid": "OK", "error": "-"})
 120             except Exception as e:
 121                 table.append({"type": desc_type, "path": desc_path, "valid": "ERROR", "error": str(e)})
 122         return table
 123
 124     def build(self, package_folder, skip_validation=False, skip_charm_build=False):
 125         """
 126             **Creates a .tar.gz file given a package_folder**
 127
 128             :params:
 129                 - package_folder: is the name of the folder to be packaged
 130                 - skip_validation: is the flag to validate or not the descriptors on the folder before build
 131
 132             :returns: message result for the build process
 133         """
 134         self._logger.debug("")
 135         package_folder = package_folder.rstrip('/')
 136         if not os.path.exists("{}".format(package_folder)):
 137             return "Fail, package is not in the specified path"
 138         if not skip_validation:
 139             print('Validating package {}'.format(package_folder))
 140             results = self.validate(package_folder, recursive=False)
 141             if results:
 142                 for result in results:
 143                     if result["valid"] != "OK":
 144                         raise ClientException("There was an error validating the file {} with error: {}"
 145                                               .format(result["path"], result["error"]))
 146                 print('Validation OK')
 147             else:
 148                 raise ClientException("No descriptor file found in: {}".format(package_folder))
 149         charm_list = self.build_all_charms(package_folder, skip_charm_build)
 150         return self.build_tarfile(package_folder, charm_list)
 151
 152     def calculate_checksum(self, package_folder):
 153         """
 154             **Function to calculate the checksum given a folder**
 155
 156             :params:
 157                 - package_folder: is the folder where we have the files to calculate the checksum
 158             :returns: None
 159         """
 160         self._logger.debug("")
 161         files = [f for f in glob.glob(package_folder + "/**/*.*", recursive=True) if os.path.isfile(f)]
 162         with open("{}/checksums.txt".format(package_folder), "w+") as checksum:
 163             for file_item in files:
 164                 if "checksums.txt" in file_item:
 165                     continue
 166                 # from https://www.quickprogrammingtips.com/python/how-to-calculate-md5-hash-of-a-file-in-python.html
 167                 md5_hash = hashlib.md5()
 168                 with open(file_item, "rb") as f:
 169                     # Read and update hash in chunks of 4K
 170                     for byte_block in iter(lambda: f.read(4096), b""):
 171                         md5_hash.update(byte_block)
 172                     checksum.write("{}\t{}\n".format(md5_hash.hexdigest(), file_item))
 173
 174     def create_folders(self, folders, package_type):
 175         """
 176             **Create folder given a list of folders**
 177
 178             :params:
 179                 - folders: [List] list of folders paths to be created
 180                 - package_type: is the type of package to be created
 181             :return: None
 182         """
 183         self._logger.debug("")
 184         for folder in folders:
 185             try:
 186                 # print("Folder {} == package_type {}".format(folder[1], package_type))
 187                 if folder[1] == package_type:
 188                     print("Creating folder:\t{}".format(folder[0]))
 189                     os.makedirs(folder[0])
 190             except FileExistsError:
 191                 pass
 192
 193     def save_file(self, file_name, file_body):
 194         """
 195             **Create a file given a name and the content**
 196
 197             :params:
 198                 - file_name: is the name of the file with the relative route
 199                 - file_body: is the content of the file
 200             :return: None
 201         """
 202         self._logger.debug("")
 203         print("Creating file:  \t{}".format(file_name))
 204         try:
 205             with open(file_name, "w+") as f:
 206                 f.write(file_body)
 207         except Exception as e:
 208             raise ClientException(e)
 209
 210     def generate_readme(self):
 211         """
 212             **Creates the README content**
 213
 214             :returns: readme content
 215         """
 216         self._logger.debug("")
 217         return """# Descriptor created by OSM descriptor package generated\n\n**Created on {} **""".format(
 218                time.strftime("%m/%d/%Y, %H:%M:%S", time.localtime()))
 219
 220     def generate_cloud_init(self):
 221         """
 222             **Creates the cloud-init content**
 223
 224             :returns: cloud-init content
 225         """
 226         self._logger.debug("")
 227         return "---\n#cloud-config"
 228
 229     def create_files(self, files, file_content, package_type):
 230         """
 231             **Creates the files given the file list and type**
 232
 233             :params:
 234                 - files: is the list of files structure
 235                 - file_content: is the content of the descriptor rendered by the template
 236                 - package_type: is the type of package to filter the creation structure
 237
 238             :return: None
 239         """
 240         self._logger.debug("")
 241         for file_item, file_package, file_type in files:
 242             if package_type == file_package:
 243                 if file_type == "descriptor":
 244                     self.save_file(file_item, file_content)
 245                 elif file_type == "readme":
 246                     self.save_file(file_item, self.generate_readme())
 247                 elif file_type == "cloud_init":
 248                     self.save_file(file_item, self.generate_cloud_init())
 249
 250     def check_files_folders(self, path_list, override):
 251         """
 252             **Find files and folders missing given a directory structure {"folders": [], "files": []}**
 253
 254             :params:
 255                 - path_list: is the list of files and folders to be created
 256                 - override: is the flag used to indicate the creation of the list even if the file exist to override it
 257
 258             :return: Missing paths Dict
 259         """
 260         self._logger.debug("")
 261         missing_paths = {}
 262         folders = []
 263         files = []
 264         for folder in path_list.get("folders"):
 265             if not os.path.exists(folder[0]):
 266                 folders.append(folder)
 267         missing_paths["folders"] = folders
 268
 269         for file_item in path_list.get("files"):
 270             if not os.path.exists(file_item[0]) or override is True:
 271                 files.append(file_item)
 272         missing_paths["files"] = files
 273
 274         return missing_paths
 275
 276     def build_all_charms(self, package_folder, skip_charm_build):
 277         """
 278             **Read the descriptor file, check that the charms referenced are in the folder and compiles them**
 279
 280             :params:
 281                 - packet_folder: is the location of the package
 282             :return: Files and Folders not found. In case of override, it will return all file list
 283         """
 284         self._logger.debug("")
 285         listCharms = []
 286         descriptor_file = False
 287         descriptors_paths = [f for f in glob.glob(package_folder + "/*.yaml")]
 288         for file in descriptors_paths:
 289             if file.endswith('nfd.yaml'):
 290                 descriptor_file = True
 291                 listCharms = self.charms_search(file, 'vnf')
 292             if file.endswith('nsd.yaml'):
 293                 descriptor_file = True
 294                 listCharms = self.charms_search(file, 'ns')
 295         print("List of charms in the descriptor: {}".format(listCharms))
 296         if not descriptor_file:
 297             raise ClientException('descriptor name is not correct in: {}'.format(package_folder))
 298         if listCharms and not skip_charm_build:
 299             for charmName in listCharms:
 300                 if os.path.isdir('{}/charms/layers/{}'.format(package_folder, charmName)):
 301                     print('Building charm {}/charms/layers/{}'.format(package_folder, charmName))
 302                     self.charm_build(package_folder, charmName)
 303                     print('Charm built: {}'.format(charmName))
 304                 else:
 305                     if not os.path.isdir('{}/charms/{}'.format(package_folder, charmName)):
 306                         raise ClientException('The charm: {} referenced in the descriptor file '
 307                                               'is not present either in {}/charms or in {}/charms/layers'.
 308                                               format(charmName, package_folder, package_folder))
 309         self._logger.debug("Return list of charms: {}".format(listCharms))
 310         return listCharms
 311
 312     def discover_folder_structure(self, base_directory, name, override):
 313         """
 314             **Discover files and folders structure for OSM descriptors given a base_directory and name**
 315
 316             :params:
 317                 - base_directory: is the location of the package to be created
 318                 - name: is the name of the package
 319                 - override: is the flag used to indicate the creation of the list even if the file exist to override it
 320             :return: Files and Folders not found. In case of override, it will return all file list
 321         """
 322         self._logger.debug("")
 323         prefix = "{}/{}".format(base_directory, name)
 324         files_folders = {"folders": [("{}_ns".format(prefix), "ns"),
 325                                      ("{}_ns/icons".format(prefix), "ns"),
 326                                      ("{}_ns/charms".format(prefix), "ns"),
 327                                      ("{}_vnf".format(name), "vnf"),
 328                                      ("{}_vnf/charms".format(prefix), "vnf"),
 329                                      ("{}_vnf/cloud_init".format(prefix), "vnf"),
 330                                      ("{}_vnf/images".format(prefix), "vnf"),
 331                                      ("{}_vnf/icons".format(prefix), "vnf"),
 332                                      ("{}_vnf/scripts".format(prefix), "vnf"),
 333                                      ("{}_nst".format(prefix), "nst"),
 334                                      ("{}_nst/icons".format(prefix), "nst")
 335                                      ],
 336                          "files": [("{}_ns/{}_nsd.yaml".format(prefix, name), "ns", "descriptor"),
 337                                    ("{}_ns/README.md".format(prefix), "ns", "readme"),
 338                                    ("{}_vnf/{}_vnfd.yaml".format(prefix, name), "vnf", "descriptor"),
 339                                    ("{}_vnf/cloud_init/cloud-config.txt".format(prefix), "vnf", "cloud_init"),
 340                                    ("{}_vnf/README.md".format(prefix), "vnf", "readme"),
 341                                    ("{}_nst/{}_nst.yaml".format(prefix, name), "nst", "descriptor"),
 342                                    ("{}_nst/README.md".format(prefix), "nst", "readme")
 343                                    ]
 344                          }
 345         missing_files_folders = self.check_files_folders(files_folders, override)
 346         # print("Missing files and folders: {}".format(missing_files_folders))
 347         return missing_files_folders
 348
 349     def charm_build(self, charms_folder, build_name):
 350         """
 351         Build the charms inside the package.
 352         params: package_folder is the name of the folder where is the charms to compile.
 353                 build_name is the name of the layer or interface
 354         """
 355         self._logger.debug("")
 356         os.environ['JUJU_REPOSITORY'] = "{}/charms".format(charms_folder)
 357         os.environ['CHARM_LAYERS_DIR'] = "{}/layers".format(os.environ['JUJU_REPOSITORY'])
 358         os.environ['CHARM_INTERFACES_DIR'] = "{}/interfaces".format(os.environ['JUJU_REPOSITORY'])
 359         os.environ['CHARM_BUILD_DIR'] = "{}/charms/builds".format(charms_folder)
 360         if not os.path.exists(os.environ['CHARM_BUILD_DIR']):
 361             os.makedirs(os.environ['CHARM_BUILD_DIR'])
 362         src_folder = '{}/{}'.format(os.environ['CHARM_LAYERS_DIR'], build_name)
 363         result = subprocess.run(["charm", "build", "{}".format(src_folder)])
 364         if result.returncode == 1:
 365             raise ClientException("failed to build the charm: {}".format(src_folder))
 366         self._logger.verbose("charm {} built".format(src_folder))
 367
 368     def build_tarfile(self, package_folder, charm_list=None):
 369         """
 370         Creates a .tar.gz file given a package_folder
 371         params: package_folder is the name of the folder to be packaged
 372         returns: .tar.gz name
 373         """
 374         self._logger.debug("")
 375         cwd = None
 376         try:
 377             directory_name, package_name = self.create_temp_dir(package_folder, charm_list)
 378             cwd = os.getcwd()
 379             os.chdir(directory_name)
 380             self.calculate_checksum(package_name)
 381             with tarfile.open("{}.tar.gz".format(package_name), mode='w:gz') as archive:
 382                 print("Adding File: {}".format(package_name))
 383                 archive.add('{}'.format(package_name), recursive=True)
 384             # return "Created {}.tar.gz".format(package_folder)
 385             # self.build("{}".format(os.path.basename(package_folder)))
 386             os.chdir(cwd)
 387             cwd = None
 388             created_package = "{}/{}.tar.gz".format(os.path.dirname(package_folder) or '.', package_name)
 389             os.rename("{}/{}.tar.gz".format(directory_name, package_name),
 390                       created_package)
 391             os.rename("{}/{}/checksums.txt".format(directory_name, package_name),
 392                       "{}/checksums.txt".format(package_folder))
 393             print("Package created: {}".format(created_package))
 394             return created_package
 395         except Exception as exc:
 396             raise ClientException('failure during build of targz file (create temp dir, calculate checksum, '
 397                                   'tar.gz file): {}'.format(exc))
 398         finally:
 399             if cwd:
 400                 os.chdir(cwd)
 401             shutil.rmtree(os.path.join(package_folder, "tmp"))
 402
 403     def create_temp_dir(self, package_folder, charm_list=None):
 404         """
 405         Method to create a temporary folder where we can move the files in package_folder
 406         """
 407         self._logger.debug("")
 408         ignore_patterns = ('.gitignore')
 409         ignore = shutil.ignore_patterns(ignore_patterns)
 410         directory_name = os.path.abspath(package_folder)
 411         package_name = os.path.basename(directory_name)
 412         directory_name += "/tmp"
 413         os.makedirs("{}/{}".format(directory_name, package_name), exist_ok=True)
 414         self._logger.debug("Makedirs DONE: {}/{}".format(directory_name, package_name))
 415         for item in os.listdir(package_folder):
 416             self._logger.debug("Item: {}".format(item))
 417             if item != "tmp":
 418                 s = os.path.join(package_folder, item)
 419                 d = os.path.join(os.path.join(directory_name, package_name), item)
 420                 if os.path.isdir(s):
 421                     if item == "charms":
 422                         os.makedirs(d, exist_ok=True)
 423                         s_builds = os.path.join(s, "builds")
 424                         for charm in charm_list:
 425                             self._logger.debug("Copying charm {}".format(charm))
 426                             if charm in os.listdir(s):
 427                                 s_charm = os.path.join(s, charm)
 428                             elif charm in os.listdir(s_builds):
 429                                 s_charm = os.path.join(s_builds, charm)
 430                             else:
 431                                 raise ClientException('The charm {} referenced in the descriptor file '
 432                                                       'could not be found in {}/charms or in {}/charms/builds'.
 433                                                       format(charm, package_folder, package_folder))
 434                             d_temp = os.path.join(d, charm)
 435                             self._logger.debug("Copying tree: {} -> {}".format(s_charm, d_temp))
 436                             shutil.copytree(s_charm, d_temp, symlinks=True, ignore=ignore)
 437                             self._logger.debug("DONE")
 438                     else:
 439                         self._logger.debug("Copying tree: {} -> {}".format(s, d))
 440                         shutil.copytree(s, d, symlinks=True, ignore=ignore)
 441                         self._logger.debug("DONE")
 442                 else:
 443                     if item in ignore_patterns:
 444                         continue
 445                     self._logger.debug("Copying file: {} -> {}".format(s, d))
 446                     shutil.copy2(s, d)
 447                     self._logger.debug("DONE")
 448         return directory_name, package_name
 449
 450     def charms_search(self, descriptor_file, desc_type):
 451         self._logger.debug("descriptor_file: {}, desc_type: {}".format(descriptor_file,
 452                                                                        desc_type))
 453         with open("{}".format(descriptor_file)) as yaml_desc:
 454             descriptor_dict = yaml.safe_load(yaml_desc)
 455             #self._logger.debug("\n"+yaml.safe_dump(descriptor_dict, indent=4, default_flow_style=False))
 456
 457             if ( (desc_type=="vnf" and ("vnfd:vnfd-catalog" in descriptor_dict or "vnfd-catalog" in descriptor_dict)) or
 458                  (desc_type=="ns" and ( "nsd:nsd-catalog" in descriptor_dict or "nsd-catalog" in descriptor_dict)) ):
 459                 charms_list = self._charms_search_on_osm_im_dict(descriptor_dict, desc_type)
 460             else:
 461                 if desc_type == "ns":
 462                     get_charm_list = self._charms_search_on_nsd_sol006_dict
 463                 elif desc_type == "vnf":
 464                     get_charm_list = self._charms_search_on_vnfd_sol006_dict
 465                 else:
 466                     raise Exception("Bad descriptor type")
 467                 charms_list = get_charm_list(descriptor_dict)
 468         return charms_list
 469
 470     def _charms_search_on_osm_im_dict(self, osm_im_dict, desc_type):
 471         self._logger.debug("")
 472         charms_list = []
 473         for k1, v1 in osm_im_dict.items():
 474             for k2, v2 in v1.items():
 475                 for entry in v2:
 476                     if '{}-configuration'.format(desc_type) in entry:
 477                         vnf_config = entry['{}-configuration'.format(desc_type)]
 478                         for k3, v3 in vnf_config.items():
 479                             if 'charm' in v3:
 480                                 charms_list.append((v3['charm']))
 481                     if 'vdu' in entry:
 482                         vdus = entry['vdu']
 483                         for vdu in vdus:
 484                             if 'vdu-configuration' in vdu:
 485                                 for k4, v4 in vdu['vdu-configuration'].items():
 486                                     if 'charm' in v4:
 487                                         charms_list.append((v4['charm']))
 488         return charms_list
 489
 490     def _charms_search_on_vnfd_sol006_dict(self, sol006_dict):
 491         self._logger.debug("")
 492         charms_list = []
 493         for k1, v1 in sol006_dict.items():
 494             for k2, v2 in v1.items():
 495                 if 'vnf-configuration' in k2:
 496                     for vnf_config in v2:
 497                         for k3, v3 in vnf_config.items():
 498                             if 'charm' in v3:
 499                                 charms_list.append((v3['charm']))
 500                 if 'vdu-configuration' in k2:
 501                     for vdu_config in v2:
 502                         for k3, v3 in vdu_config.items():
 503                             if 'charm' in v3:
 504                                 charms_list.append((v3['charm']))
 505         return charms_list
 506
 507     def _charms_search_on_nsd_sol006_dict(self, sol006_dict):
 508         self._logger.debug("")
 509         charms_list = []
 510         nsd_list = sol006_dict.get("nsd", {}).get("nsd", [])
 511         for nsd in nsd_list:
 512             charm = nsd.get("ns-configuration", {}).get("juju", {}).get("charm")
 513             if charm:
 514                 charms_list.append(charm)
 515         return charms_list