import hashlib
from osm_im.validation import Validation as validation_im
from jinja2 import Environment, PackageLoader
-
+import subprocess
+import shutil
+import yaml
+import logging
class PackageTool(object):
def __init__(self, client=None):
self._client = client
+ self._logger = logging.getLogger('osmclient')
def create(self, package_type, base_directory, package_name, override, image, vdus, vcpu, memory, storage,
interfaces, vendor, detailed, netslice_subnets, netslice_vlds):
:return: status
"""
-
+ self._logger.debug("")
# print("location: {}".format(osmclient.__path__))
file_loader = PackageLoader("osmclient")
env = Environment(loader=file_loader)
self.create_files(structure["files"], output, package_type)
return "Created"
- def validate(self, base_directory):
+ def validate(self, base_directory, recursive=True):
"""
**Validate OSM Descriptors given a path**
:return: List of dict of validated descriptors. keys: type, path, valid, error
"""
+ self._logger.debug("")
table = []
- descriptors_paths = [f for f in glob.glob(base_directory + "/**/*.yaml", recursive=True)]
+ if recursive:
+ descriptors_paths = [f for f in glob.glob(base_directory + "/**/*.yaml", recursive=recursive)]
+ else:
+ descriptors_paths = [f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)]
print("Base directory: {}".format(base_directory))
print("{} Descriptors found to validate".format(len(descriptors_paths)))
for desc_path in descriptors_paths:
table.append({"type": desc_type, "path": desc_path, "valid": "ERROR", "error": str(e)})
return table
- def build(self, package_folder, skip_validation=True):
+ def build(self, package_folder, skip_validation=False, skip_charm_build=False):
"""
**Creates a .tar.gz file given a package_folder**
:returns: message result for the build process
"""
-
+ self._logger.debug("")
+ package_folder = package_folder.rstrip('/')
if not os.path.exists("{}".format(package_folder)):
- return "Fail, package is not in the specified route"
+ return "Fail, package is not in the specified path"
if not skip_validation:
- results = self.validate(package_folder)
- for result in results:
- if result["valid"] != "OK":
- return("There was an error validating the file: {} with error: {}".format(result["path"],
- result["error"]))
- self.calculate_checksum(package_folder)
- with tarfile.open("{}.tar.gz".format(package_folder), mode='w:gz') as archive:
- print("Adding File: {}".format(package_folder))
- archive.add('{}'.format(package_folder), recursive=True)
- return "Created {}.tar.gz".format(package_folder)
+ print('Validating package {}'.format(package_folder))
+ results = self.validate(package_folder, recursive=False)
+ if results:
+ for result in results:
+ if result["valid"] != "OK":
+ raise ClientException("There was an error validating the file {} with error: {}"
+ .format(result["path"], result["error"]))
+ print('Validation OK')
+ else:
+ raise ClientException("No descriptor file found in: {}".format(package_folder))
+ charm_list = self.build_all_charms(package_folder, skip_charm_build)
+ return self.build_tarfile(package_folder, charm_list)
def calculate_checksum(self, package_folder):
"""
- package_folder: is the folder where we have the files to calculate the checksum
:returns: None
"""
- files = [f for f in glob.glob(package_folder + "/**/*.*", recursive=True)]
- checksum = open("{}/checksum.txt".format(package_folder), "w+")
- for file_item in files:
- if "checksum.txt" in file_item:
- continue
- # from https://www.quickprogrammingtips.com/python/how-to-calculate-md5-hash-of-a-file-in-python.html
- md5_hash = hashlib.md5()
- with open(file_item, "rb") as f:
- # Read and update hash in chunks of 4K
- for byte_block in iter(lambda: f.read(4096), b""):
- md5_hash.update(byte_block)
- checksum.write("{}\t{}\n".format(md5_hash.hexdigest(), file_item))
- checksum.close()
+ self._logger.debug("")
+ files = [f for f in glob.glob(package_folder + "/**/*.*", recursive=True) if os.path.isfile(f)]
+ with open("{}/checksums.txt".format(package_folder), "w+") as checksum:
+ for file_item in files:
+ if "checksums.txt" in file_item:
+ continue
+ # from https://www.quickprogrammingtips.com/python/how-to-calculate-md5-hash-of-a-file-in-python.html
+ md5_hash = hashlib.md5()
+ with open(file_item, "rb") as f:
+ # Read and update hash in chunks of 4K
+ for byte_block in iter(lambda: f.read(4096), b""):
+ md5_hash.update(byte_block)
+ checksum.write("{}\t{}\n".format(md5_hash.hexdigest(), file_item))
def create_folders(self, folders, package_type):
"""
:return: None
"""
+ self._logger.debug("")
for file_item, file_package, file_type in files:
if package_type == file_package:
if file_type == "descriptor":
:return: Missing paths Dict
"""
+ self._logger.debug("")
missing_paths = {}
folders = []
files = []
return missing_paths
+ def build_all_charms(self, package_folder, skip_charm_build):
+ """
+ **Read the descriptor file, check that the charms referenced are in the folder and compiles them**
+
+ :params:
+ - packet_folder: is the location of the package
+ :return: Files and Folders not found. In case of override, it will return all file list
+ """
+ self._logger.debug("")
+ listCharms = []
+ descriptor_file = False
+ descriptors_paths = [f for f in glob.glob(package_folder + "/*.yaml")]
+ for file in descriptors_paths:
+ if file.endswith('nfd.yaml'):
+ descriptor_file = True
+ listCharms = self.charms_search(file, 'vnf')
+ if file.endswith('nsd.yaml'):
+ descriptor_file = True
+ listCharms = self.charms_search(file, 'ns')
+ print("List of charms in the descriptor: {}".format(listCharms))
+ if not descriptor_file:
+ raise ClientException ('descriptor name is not correct in: {}'.format(package_folder))
+ if listCharms and not skip_charm_build:
+ for charmName in listCharms:
+ if os.path.isdir('{}/charms/layers/{}'.format(package_folder,charmName)):
+ print('Building charm {}/charms/layers/{}'.format(package_folder, charmName))
+ self.charm_build(package_folder, charmName)
+ print('Charm built: {}'.format(charmName))
+ else:
+ if not os.path.isdir('{}/charms/{}'.format(package_folder,charmName)):
+ raise ClientException ('The charm: {} referenced in the descriptor file '
+ 'is not present either in {}/charms or in {}/charms/layers'.
+ format(charmName, package_folder,package_folder))
+ self._logger.debug("Return list of charms: {}".format(listCharms))
+ return listCharms
+
def discover_folder_structure(self, base_directory, name, override):
"""
**Discover files and folders structure for OSM descriptors given a base_directory and name**
- override: is the flag used to indicate the creation of the list even if the file exist to override it
:return: Files and Folders not found. In case of override, it will return all file list
"""
+ self._logger.debug("")
prefix = "{}/{}".format(base_directory, name)
files_folders = {"folders": [("{}_ns".format(prefix), "ns"),
("{}_ns/icons".format(prefix), "ns"),
missing_files_folders = self.check_files_folders(files_folders, override)
# print("Missing files and folders: {}".format(missing_files_folders))
return missing_files_folders
+
+ def charm_build(self, charms_folder, build_name):
+ """
+ Build the charms inside the package.
+ params: package_folder is the name of the folder where is the charms to compile.
+ build_name is the name of the layer or interface
+ """
+ self._logger.debug("")
+ os.environ['JUJU_REPOSITORY'] = "{}/charms".format(charms_folder)
+ os.environ['CHARM_LAYERS_DIR'] = "{}/layers".format(os.environ['JUJU_REPOSITORY'])
+ os.environ['CHARM_INTERFACES_DIR'] = "{}/interfaces".format(os.environ['JUJU_REPOSITORY'])
+ os.environ['CHARM_BUILD_DIR'] = "{}/charms/builds".format(charms_folder)
+ if not os.path.exists(os.environ['CHARM_BUILD_DIR']):
+ os.makedirs(os.environ['CHARM_BUILD_DIR'])
+ src_folder = '{}/{}'.format(os.environ['CHARM_LAYERS_DIR'], build_name)
+ result = subprocess.run(["charm", "build", "{}".format(src_folder)])
+ if result.returncode == 1:
+ raise ClientException("failed to build the charm: {}".format(src_folder))
+ self._logger.verbose("charm {} built".format(src_folder))
+
+ def build_tarfile(self, package_folder, charm_list=None):
+ """
+ Creates a .tar.gz file given a package_folder
+ params: package_folder is the name of the folder to be packaged
+ returns: .tar.gz name
+ """
+ self._logger.debug("")
+ cwd = None
+ try:
+ directory_name, package_name = self.create_temp_dir(package_folder, charm_list)
+ cwd = os.getcwd()
+ os.chdir(directory_name)
+ self.calculate_checksum(package_name)
+ with tarfile.open("{}.tar.gz".format(package_name), mode='w:gz') as archive:
+ print("Adding File: {}".format(package_name))
+ archive.add('{}'.format(package_name), recursive=True)
+ # return "Created {}.tar.gz".format(package_folder)
+ # self.build("{}".format(os.path.basename(package_folder)))
+ os.chdir(cwd)
+ cwd = None
+ created_package = "{}/{}.tar.gz".format(os.path.dirname(package_folder) or '.', package_name)
+ os.rename("{}/{}.tar.gz".format(directory_name, package_name),
+ created_package)
+ os.rename("{}/{}/checksums.txt".format(directory_name, package_name),
+ "{}/checksums.txt".format(package_folder))
+ print("Package created: {}".format(created_package))
+ return created_package
+ except Exception as exc:
+ raise ClientException('failure during build of targz file (create temp dir, calculate checksum, '
+ 'tar.gz file): {}'.format(exc))
+ finally:
+ if cwd:
+ os.chdir(cwd)
+ shutil.rmtree(os.path.join(package_folder, "tmp"))
+
+ def create_temp_dir(self, package_folder, charm_list=None):
+ """
+ Method to create a temporary folder where we can move the files in package_folder
+ """
+ self._logger.debug("")
+ ignore_patterns = ('.gitignore')
+ ignore = shutil.ignore_patterns(ignore_patterns)
+ directory_name = os.path.abspath(package_folder)
+ package_name = os.path.basename(directory_name)
+ directory_name += "/tmp"
+ os.makedirs("{}/{}".format(directory_name, package_name), exist_ok=True)
+ self._logger.debug("Makedirs DONE: {}/{}".format(directory_name, package_name))
+ for item in os.listdir(package_folder):
+ self._logger.debug("Item: {}".format(item))
+ if item != "tmp":
+ s = os.path.join(package_folder, item)
+ d = os.path.join(os.path.join(directory_name, package_name), item)
+ if os.path.isdir(s):
+ if item == "charms":
+ os.makedirs(d, exist_ok=True)
+ s_builds = os.path.join(s, "builds")
+ for charm in charm_list:
+ self._logger.debug("Copying charm {}".format(charm))
+ if charm in os.listdir(s):
+ s_charm = os.path.join(s, charm)
+ elif charm in os.listdir(s_builds):
+ s_charm = os.path.join(s_builds, charm)
+ else:
+ raise ClientException('The charm {} referenced in the descriptor file '
+ 'could not be found in {}/charms or in {}/charms/builds'.
+ format(charm, package_folder, package_folder))
+ d_temp = os.path.join(d, charm)
+ self._logger.debug("Copying tree: {} -> {}".format(s_charm, d_temp))
+ shutil.copytree(s_charm, d_temp, symlinks=True, ignore=ignore)
+ self._logger.debug("DONE")
+ else:
+ self._logger.debug("Copying tree: {} -> {}".format(s,d))
+ shutil.copytree(s, d, symlinks=True, ignore=ignore)
+ self._logger.debug("DONE")
+ else:
+ if item in ignore_patterns:
+ continue
+ self._logger.debug("Copying file: {} -> {}".format(s,d))
+ shutil.copy2(s, d)
+ self._logger.debug("DONE")
+ return directory_name, package_name
+
+ def charms_search(self, descriptor_file, desc_type):
+ self._logger.debug("")
+ dict = {}
+ list = []
+ with open("{}".format(descriptor_file)) as yaml_desc:
+ dict = yaml.safe_load(yaml_desc)
+ for k1, v1 in dict.items():
+ for k2, v2 in v1.items():
+ for entry in v2:
+ if '{}-configuration'.format(desc_type) in entry:
+ name = entry['{}-configuration'.format(desc_type)]
+ for k3, v3 in name.items():
+ if 'charm' in v3:
+ list.append((v3['charm']))
+ if 'vdu' in entry:
+ name = entry['vdu']
+ for vdu in name:
+ if 'vdu-configuration' in vdu:
+ for k4, v4 in vdu['vdu-configuration'].items():
+ if 'charm' in v4:
+ list.append((v4['charm']))
+ return list
+
projects / osm / osmclient.git / blobdiff