# License for the specific language governing permissions and limitations
# under the License.
-from osmclient.common.exceptions import ClientException
-import os
import glob
-import time
-import tarfile
import hashlib
-from osm_im.validation import Validation as validation_im
-from jinja2 import Environment, PackageLoader
-import subprocess
+import logging
+import os
import shutil
+import subprocess
+import tarfile
+import time
+
+from jinja2 import Environment, PackageLoader
+from osm_im.validation import Validation as validation_im
+from osm_im.validation import ValidationException
+from osm_im import im_translation
+from osmclient.common.exceptions import ClientException
import yaml
-import logging
+
class PackageTool(object):
def __init__(self, client=None):
self._client = client
self._logger = logging.getLogger('osmclient')
+ self._validator = validation_im()
def create(self, package_type, base_directory, package_name, override, image, vdus, vcpu, memory, storage,
- interfaces, vendor, detailed, netslice_subnets, netslice_vlds):
+ interfaces, vendor, detailed, netslice_subnets, netslice_vlds, old):
"""
**Create a package descriptor**
- detailed: include all possible values for NSD, VNFD, NST
- netslice_subnets: number of netslice_subnets for the NST
- netslice_vlds: number of virtual link descriptors for the NST
+ - old: flag to create a package using the IM of OSM<9
:return: status
"""
file_loader = PackageLoader("osmclient")
env = Environment(loader=file_loader)
if package_type == 'ns':
- template = env.get_template('nsd.yaml.j2')
+ template = env.get_template("nsd.yaml.j2" if not old else "nsd_old.yaml.j2")
content = {"name": package_name, "vendor": vendor, "vdus": vdus, "clean": False, "interfaces": interfaces,
"detailed": detailed}
elif package_type == 'vnf':
- template = env.get_template('vnfd.yaml.j2')
+ template = env.get_template("vnfd.yaml.j2" if not old else "vnfd_old.yaml.j2")
content = {"name": package_name, "vendor": vendor, "vdus": vdus, "clean": False, "interfaces": interfaces,
"image": image, "vcpu": vcpu, "memory": memory, "storage": storage, "detailed": detailed}
elif package_type == 'nst':
self.create_files(structure["files"], output, package_type)
return "Created"
- def validate(self, base_directory, recursive=True):
+ def validate(self, base_directory, recursive=True, old_format=False):
"""
**Validate OSM Descriptors given a path**
desc_type = "-"
try:
desc_type, descriptor_data = validation_im.yaml_validation(self, descriptor_data)
+ if not old_format:
+ if ( desc_type=="vnfd" or desc_type=="nsd" ):
+ print("OSM descriptor '{}' written in an unsupported format. Please update to ETSI SOL006 format".format(desc_path))
+ print("Package validation skipped. It can still be done with 'osm package-validate --old'")
+ print("Package build can still be done with 'osm package-build --skip-validation'")
+ raise Exception("Not SOL006 format")
validation_im.pyangbind_validation(self, desc_type, descriptor_data)
table.append({"type": desc_type, "path": desc_path, "valid": "OK", "error": "-"})
except Exception as e:
table.append({"type": desc_type, "path": desc_path, "valid": "ERROR", "error": str(e)})
return table
+ def translate(self, base_directory, recursive=True, dryrun=False):
+ """
+ **Translate OSM Packages given a path**
+
+ :params:
+ - base_directory is the root path for all packages
+
+ :return: List of dict of translated packages. keys: current type, new type, path, valid, translated, error
+ """
+ self._logger.debug("")
+ table = []
+ if recursive:
+ descriptors_paths = [f for f in glob.glob(base_directory + "/**/*.yaml", recursive=recursive)]
+ else:
+ descriptors_paths = [f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)]
+ print("Base directory: {}".format(base_directory))
+ print("{} Descriptors found to validate".format(len(descriptors_paths)))
+ for desc_path in descriptors_paths:
+ with open(desc_path) as descriptor_file:
+ descriptor_data = descriptor_file.read()
+ desc_type = "-"
+ try:
+ desc_type, descriptor_data = validation_im.yaml_validation(self, descriptor_data)
+ self._logger.debug("desc_type: {}".format(desc_type))
+ self._logger.debug("descriptor_data:\n{}".format(descriptor_data))
+ self._validator.pyangbind_validation(desc_type, descriptor_data)
+ if not ( desc_type=="vnfd" or desc_type=="nsd" ):
+ table.append({"current type": desc_type, "new type": desc_type, "path": desc_path, "valid": "OK", "translated": "N/A", "error": "-"})
+ else:
+ new_desc_type = desc_type
+ try:
+ sol006_model = yaml.safe_dump(im_translation.translate_im_model_to_sol006(descriptor_data), indent=4, default_flow_style=False)
+ new_desc_type, new_descriptor_data = self._validator.yaml_validation(sol006_model)
+ self._validator.pyangbind_validation(new_desc_type, new_descriptor_data)
+ if not dryrun:
+ with open(desc_path, 'w') as descriptor_file:
+ descriptor_file.write(sol006_model)
+ table.append({"current type": desc_type, "new type": new_desc_type, "path": desc_path, "valid": "OK", "translated": "OK", "error": "-"})
+ except ValidationException as ve2:
+ table.append({"current type": desc_type, "new type": new_desc_type, "path": desc_path, "valid": "OK", "translated": "ERROR", "error": "Error in the post-validation: {}".format(str(ve2))})
+ except Exception as e2:
+ table.append({"current type": desc_type, "new type": new_desc_type, "path": desc_path, "valid": "OK", "translated": "ERROR", "error": "Error in the translation: {}".format(str(e2))})
+ except ValidationException as ve:
+ table.append({"current type": desc_type, "new type": "N/A", "path": desc_path, "valid": "ERROR", "translated": "N/A", "error": "Error in the pre-validation: {}".format(str(ve))})
+ except Exception as e:
+ table.append({"current type": desc_type, "new type": "N/A", "path": desc_path, "valid": "ERROR", "translated": "N/A", "error": str(e)})
+ return table
+
+ def descriptor_translate(self, descriptor_file):
+ """
+ **Translate input descriptor file from Rel EIGHT OSM to SOL006**
+
+ :params:
+ - base_directory is the root path for all packages
+
+ :return: YAML descriptor in the new format
+ """
+ self._logger.debug("")
+ with open(descriptor_file, 'r') as df:
+ im_model = yaml.safe_load(df.read())
+ sol006_model = im_translation.translate_im_model_to_sol006(im_model)
+ return yaml.safe_dump(sol006_model, indent=4, default_flow_style=False)
+
def build(self, package_folder, skip_validation=False, skip_charm_build=False):
"""
**Creates a .tar.gz file given a package_folder**
:returns: None
"""
self._logger.debug("")
- files = [f for f in glob.glob(package_folder + "/**/*.*", recursive=True)]
- checksum = open("{}/checksums.txt".format(package_folder), "w+")
- for file_item in files:
- if "checksums.txt" in file_item:
- continue
- # from https://www.quickprogrammingtips.com/python/how-to-calculate-md5-hash-of-a-file-in-python.html
- md5_hash = hashlib.md5()
- with open(file_item, "rb") as f:
- # Read and update hash in chunks of 4K
- for byte_block in iter(lambda: f.read(4096), b""):
- md5_hash.update(byte_block)
- checksum.write("{}\t{}\n".format(md5_hash.hexdigest(), file_item))
- checksum.close()
+ files = [f for f in glob.glob(package_folder + "/**/*.*", recursive=True) if os.path.isfile(f)]
+ with open("{}/checksums.txt".format(package_folder), "w+") as checksum:
+ for file_item in files:
+ if "checksums.txt" in file_item:
+ continue
+ # from https://www.quickprogrammingtips.com/python/how-to-calculate-md5-hash-of-a-file-in-python.html
+ md5_hash = hashlib.md5()
+ with open(file_item, "rb") as f:
+ # Read and update hash in chunks of 4K
+ for byte_block in iter(lambda: f.read(4096), b""):
+ md5_hash.update(byte_block)
+ checksum.write("{}\t{}\n".format(md5_hash.hexdigest(), file_item))
def create_folders(self, folders, package_type):
"""
- package_type: is the type of package to be created
:return: None
"""
-
+ self._logger.debug("")
for folder in folders:
try:
# print("Folder {} == package_type {}".format(folder[1], package_type))
- file_body: is the content of the file
:return: None
"""
+ self._logger.debug("")
print("Creating file: \t{}".format(file_name))
try:
with open(file_name, "w+") as f:
:returns: readme content
"""
+ self._logger.debug("")
return """# Descriptor created by OSM descriptor package generated\n\n**Created on {} **""".format(
time.strftime("%m/%d/%Y, %H:%M:%S", time.localtime()))
:returns: cloud-init content
"""
+ self._logger.debug("")
return "---\n#cloud-config"
def create_files(self, files, file_content, package_type):
listCharms = self.charms_search(file, 'ns')
print("List of charms in the descriptor: {}".format(listCharms))
if not descriptor_file:
- raise ClientException ('descriptor name is not correct in: {}'.format(package_folder))
+ raise ClientException('Descriptor filename is not correct in: {}. It should end with "nfd.yaml" or "nsd.yaml"'.format(package_folder))
if listCharms and not skip_charm_build:
for charmName in listCharms:
- if os.path.isdir('{}/charms/layers/{}'.format(package_folder,charmName)):
+ if os.path.isdir('{}/charms/layers/{}'.format(package_folder, charmName)):
print('Building charm {}/charms/layers/{}'.format(package_folder, charmName))
self.charm_build(package_folder, charmName)
- print('Charm built'.format(charmName))
+ print('Charm built: {}'.format(charmName))
else:
- if not os.path.isdir('{}/charms/{}'.format(package_folder,charmName)):
- raise ClientException ('The charm: {} referenced in the descriptor file '
- 'is not present either in {}/charms or in {}/charms/layers'.
- format(charmName, package_folder,package_folder))
+ if not os.path.isdir('{}/charms/{}'.format(package_folder, charmName)):
+ raise ClientException('The charm: {} referenced in the descriptor file '
+ 'is not present either in {}/charms or in {}/charms/layers'.
+ format(charmName, package_folder, package_folder))
self._logger.debug("Return list of charms: {}".format(listCharms))
return listCharms
returns: .tar.gz name
"""
self._logger.debug("")
+ cwd = None
try:
- directory_name = self.create_temp_dir(package_folder, charm_list)
+ directory_name, package_name = self.create_temp_dir(package_folder, charm_list)
cwd = os.getcwd()
os.chdir(directory_name)
- self.calculate_checksum(package_folder)
- with tarfile.open("{}.tar.gz".format(package_folder), mode='w:gz') as archive:
- print("Adding File: {}".format(package_folder))
- archive.add('{}'.format(package_folder), recursive=True)
- #return "Created {}.tar.gz".format(package_folder)
- #self.build("{}".format(os.path.basename(package_folder)))
+ self.calculate_checksum(package_name)
+ with tarfile.open("{}.tar.gz".format(package_name), mode='w:gz') as archive:
+ print("Adding File: {}".format(package_name))
+ archive.add('{}'.format(package_name), recursive=True)
+ # return "Created {}.tar.gz".format(package_folder)
+ # self.build("{}".format(os.path.basename(package_folder)))
os.chdir(cwd)
+ cwd = None
+ created_package = "{}/{}.tar.gz".format(os.path.dirname(package_folder) or '.', package_name)
+ os.rename("{}/{}.tar.gz".format(directory_name, package_name),
+ created_package)
+ os.rename("{}/{}/checksums.txt".format(directory_name, package_name),
+ "{}/checksums.txt".format(package_folder))
+ print("Package created: {}".format(created_package))
+ return created_package
except Exception as exc:
+ raise ClientException('failure during build of targz file (create temp dir, calculate checksum, '
+ 'tar.gz file): {}'.format(exc))
+ finally:
+ if cwd:
+ os.chdir(cwd)
shutil.rmtree(os.path.join(package_folder, "tmp"))
- raise ClientException('failure during build of targz file (create temp dir, calculate checksum, tar.gz file): {}'.format(exc))
- os.rename("{}/{}.tar.gz".format(directory_name, os.path.basename(package_folder)),
- "{}.tar.gz".format(os.path.basename(package_folder)))
- os.rename("{}/{}/checksums.txt".format(directory_name, os.path.basename(package_folder)),
- "{}/checksums.txt".format(package_folder))
- shutil.rmtree(os.path.join(package_folder, "tmp"))
- print("Package created: {}.tar.gz".format(os.path.basename(package_folder)))
- return "{}.tar.gz".format(package_folder)
def create_temp_dir(self, package_folder, charm_list=None):
"""
self._logger.debug("")
ignore_patterns = ('.gitignore')
ignore = shutil.ignore_patterns(ignore_patterns)
- directory_name = os.path.abspath("{}/tmp".format(package_folder))
- os.makedirs("{}/{}".format(directory_name, os.path.basename(package_folder),exist_ok=True))
- self._logger.debug("Makedirs DONE: {}/{}".format(directory_name, os.path.basename(package_folder)))
+ directory_name = os.path.abspath(package_folder)
+ package_name = os.path.basename(directory_name)
+ directory_name += "/tmp"
+ os.makedirs("{}/{}".format(directory_name, package_name), exist_ok=True)
+ self._logger.debug("Makedirs DONE: {}/{}".format(directory_name, package_name))
for item in os.listdir(package_folder):
self._logger.debug("Item: {}".format(item))
if item != "tmp":
s = os.path.join(package_folder, item)
- d = os.path.join(os.path.join(directory_name, os.path.basename(package_folder)), item)
+ d = os.path.join(os.path.join(directory_name, package_name), item)
if os.path.isdir(s):
if item == "charms":
os.makedirs(d, exist_ok=True)
format(charm, package_folder, package_folder))
d_temp = os.path.join(d, charm)
self._logger.debug("Copying tree: {} -> {}".format(s_charm, d_temp))
- shutil.copytree(s_charm, d_temp, symlinks = True, ignore = ignore)
+ shutil.copytree(s_charm, d_temp, symlinks=True, ignore=ignore)
self._logger.debug("DONE")
else:
- self._logger.debug("Copying tree: {} -> {}".format(s,d))
- shutil.copytree(s, d, symlinks = True, ignore = ignore)
+ self._logger.debug("Copying tree: {} -> {}".format(s, d))
+ shutil.copytree(s, d, symlinks=True, ignore=ignore)
self._logger.debug("DONE")
else:
if item in ignore_patterns:
continue
- self._logger.debug("Copying file: {} -> {}".format(s,d))
+ self._logger.debug("Copying file: {} -> {}".format(s, d))
shutil.copy2(s, d)
self._logger.debug("DONE")
- return directory_name
+ return directory_name, package_name
def charms_search(self, descriptor_file, desc_type):
- self._logger.debug("")
- dict = {}
- list = []
+ self._logger.debug("descriptor_file: {}, desc_type: {}".format(descriptor_file,
+ desc_type))
with open("{}".format(descriptor_file)) as yaml_desc:
- dict = yaml.safe_load(yaml_desc)
- for k1, v1 in dict.items():
- for k2, v2 in v1.items():
- for entry in v2:
- if '{}-configuration'.format(desc_type) in entry:
- name = entry['{}-configuration'.format(desc_type)]
- for k3, v3 in name.items():
- if 'charm' in v3:
- list.append((v3['charm']))
- if 'vdu' in entry:
- name = entry['vdu']
- for vdu in name:
- if 'vdu-configuration' in vdu:
- for k4, v4 in vdu['vdu-configuration'].items():
- if 'charm' in v4:
- list.append((v4['charm']))
- return list
+ descriptor_dict = yaml.safe_load(yaml_desc)
+ #self._logger.debug("\n"+yaml.safe_dump(descriptor_dict, indent=4, default_flow_style=False))
+
+ if ( (desc_type=="vnf" and ("vnfd:vnfd-catalog" in descriptor_dict or "vnfd-catalog" in descriptor_dict)) or
+ (desc_type=="ns" and ( "nsd:nsd-catalog" in descriptor_dict or "nsd-catalog" in descriptor_dict)) ):
+ charms_list = self._charms_search_on_osm_im_dict(descriptor_dict, desc_type)
+ else:
+ if desc_type == "ns":
+ get_charm_list = self._charms_search_on_nsd_sol006_dict
+ elif desc_type == "vnf":
+ get_charm_list = self._charms_search_on_vnfd_sol006_dict
+ else:
+ raise Exception("Bad descriptor type")
+ charms_list = get_charm_list(descriptor_dict)
+ return charms_list
+ def _charms_search_on_osm_im_dict(self, osm_im_dict, desc_type):
+ self._logger.debug("")
+ charms_list = []
+ for k1, v1 in osm_im_dict.items():
+ for k2, v2 in v1.items():
+ for entry in v2:
+ if '{}-configuration'.format(desc_type) in entry:
+ vnf_config = entry['{}-configuration'.format(desc_type)]
+ for k3, v3 in vnf_config.items():
+ if 'charm' in v3:
+ charms_list.append((v3['charm']))
+ if 'vdu' in entry:
+ vdus = entry['vdu']
+ for vdu in vdus:
+ if 'vdu-configuration' in vdu:
+ for k4, v4 in vdu['vdu-configuration'].items():
+ if 'charm' in v4:
+ charms_list.append((v4['charm']))
+ return charms_list
+
+ def _charms_search_on_vnfd_sol006_dict(self, sol006_dict):
+ self._logger.debug("")
+ charms_list = []
+ for k1, v1 in sol006_dict.items():
+ for k2, v2 in v1.items():
+ if k2 == "df":
+ for df in v2:
+ lcm_ops = df.get("lcm-operations-configuration", {})
+ ops_config = lcm_ops.get("operate-vnf-op-config", {})
+ for day_12_config in ops_config.get("day1-2", []):
+ self._logger.debug("Execution environment found")
+ for ee in day_12_config.get("execution-environment-list", []):
+ if "juju" in ee:
+ charms_list.append((ee["juju"]['charm']))
+ return charms_list
+
+ def _charms_search_on_nsd_sol006_dict(self, sol006_dict):
+ self._logger.debug("")
+ charms_list = []
+ nsd_list = sol006_dict.get("nsd", {}).get("nsd", [])
+ for nsd in nsd_list:
+ charm = nsd.get("ns-configuration", {}).get("juju", {}).get("charm")
+ if charm:
+ charms_list.append(charm)
+ return charms_list