import subprocess
import tarfile
import time
-
from jinja2 import Environment, PackageLoader
from osm_im.validation import Validation as validation_im
from osm_im.validation import ValidationException
from osm_im import im_translation
+from osmclient.common import package_handling as package_handling
from osmclient.common.exceptions import ClientException
+from .sol004_package import SOL004Package
+from .sol007_package import SOL007Package
import yaml
detailed,
netslice_subnets,
netslice_vlds,
+ old,
):
"""
**Create a package descriptor**
- detailed: include all possible values for NSD, VNFD, NST
- netslice_subnets: number of netslice_subnets for the NST
- netslice_vlds: number of virtual link descriptors for the NST
+ - old: flag to create a descriptor using the previous OSM format (pre SOL006, OSM<9)
:return: status
"""
file_loader = PackageLoader("osmclient")
env = Environment(loader=file_loader)
if package_type == "ns":
- template = env.get_template("nsd.yaml.j2")
+ template = env.get_template("nsd.yaml.j2" if not old else "nsd_old.yaml.j2")
content = {
"name": package_name,
"vendor": vendor,
"detailed": detailed,
}
elif package_type == "vnf":
- template = env.get_template("vnfd.yaml.j2")
+ template = env.get_template(
+ "vnfd.yaml.j2" if not old else "vnfd_old.yaml.j2"
+ )
content = {
"name": package_name,
"vendor": vendor,
"detailed": detailed,
}
elif package_type == "nst":
+ # TODO: repo-index did not support nst in OSM<9, no changes in template
template = env.get_template("nst.yaml.j2")
content = {
"name": package_name,
"Wrong descriptor type {}. Options: ns, vnf, nst".format(package_type)
)
- # print("To be rendered: {}".format(content))
+ self._logger.debug("To be rendered: {}".format(content))
output = template.render(content)
- # print(output)
+ self._logger.debug(output)
structure = self.discover_folder_structure(
base_directory, package_name, override
descriptors_paths = [
f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)
]
- print("Base directory: {}".format(base_directory))
- print("{} Descriptors found to validate".format(len(descriptors_paths)))
+ self._logger.info("Base directory: {}".format(base_directory))
+ self._logger.info(
+ "{} Descriptors found to validate".format(len(descriptors_paths))
+ )
for desc_path in descriptors_paths:
with open(desc_path) as descriptor_file:
descriptor_data = descriptor_file.read()
desc_type = "-"
try:
+ # TODO: refactor validation_im.yaml_validation to @staticmethod
desc_type, descriptor_data = validation_im.yaml_validation(
self, descriptor_data
)
+ self._logger.debug(f"Validate {desc_type} {descriptor_data}")
if not old_format:
if desc_type == "vnfd" or desc_type == "nsd":
- print(
+ self._logger.error(
"OSM descriptor '{}' written in an unsupported format. Please update to ETSI SOL006 format".format(
desc_path
)
)
- print(
+ self._logger.warning(
"Package validation skipped. It can still be done with 'osm package-validate --old'"
)
- print(
+ self._logger.warning(
"Package build can still be done with 'osm package-build --skip-validation'"
)
raise Exception("Not SOL006 format")
{"type": desc_type, "path": desc_path, "valid": "OK", "error": "-"}
)
except Exception as e:
+ self._logger.error(f"Validation error: {e}", exc_info=True)
table.append(
{
"type": desc_type,
"error": str(e),
}
)
+ self._logger.debug(table[-1])
return table
def translate(self, base_directory, recursive=True, dryrun=False):
raise ClientException(
"No descriptor file found in: {}".format(package_folder)
)
- charm_list = self.build_all_charms(package_folder, skip_charm_build)
- return self.build_tarfile(package_folder, charm_list)
+
+ is_sol004_007 = (
+ package_handling.get_package_type(package_folder)
+ != package_handling.OSM_OLD
+ )
+
+ charm_list = self.build_all_charms(
+ package_folder, skip_charm_build, is_sol004_007
+ )
+ return self.build_compressed_file(package_folder, charm_list, is_sol004_007)
def calculate_checksum(self, package_folder):
"""
return missing_paths
- def build_all_charms(self, package_folder, skip_charm_build):
+ def build_all_charms(self, package_folder, skip_charm_build, sol004_007=True):
"""
**Read the descriptor file, check that the charms referenced are in the folder and compiles them**
self._logger.debug("")
charms_set = set()
descriptor_file = False
- descriptors_paths = [f for f in glob.glob(package_folder + "/*.yaml")]
+ package_type = package_handling.get_package_type(package_folder)
+ if sol004_007 and package_type.find("TOSCA") >= 0:
+ descriptors_paths = [
+ f for f in glob.glob(package_folder + "/Definitions/*.yaml")
+ ]
+ else:
+ descriptors_paths = [f for f in glob.glob(package_folder + "/*.yaml")]
for file in descriptors_paths:
if file.endswith("nfd.yaml"):
descriptor_file = True
if charms_set and not skip_charm_build:
for charmName in charms_set:
if os.path.isdir(
- "{}/charms/layers/{}".format(package_folder, charmName)
+ "{}/{}charms/layers/{}".format(
+ package_folder, "Scripts/" if sol004_007 else "", charmName
+ )
):
print(
- "Building charm {}/charms/layers/{}".format(
- package_folder, charmName
+ "Building charm {}/{}charms/layers/{}".format(
+ package_folder, "Scripts/" if sol004_007 else "", charmName
)
)
- self.charm_build(package_folder, charmName)
+ self.charm_build(package_folder, charmName, sol004_007)
print("Charm built: {}".format(charmName))
elif os.path.isdir(
- "{}/charms/ops/{}".format(package_folder, charmName)
+ "{}/{}charms/ops/{}".format(
+ package_folder, "Scripts/" if sol004_007 else "", charmName
+ )
):
self.charmcraft_build(package_folder, charmName)
else:
if not os.path.isdir(
- "{}/charms/{}".format(package_folder, charmName)
+ "{}/{}charms/{}".format(
+ package_folder, "Scripts/" if sol004_007 else "", charmName
+ )
+ ) and not os.path.isfile(
+ "{}/{}charms/{}".format(
+ package_folder, "Scripts/" if sol004_007 else "", charmName
+ )
):
raise ClientException(
"The charm: {} referenced in the descriptor file "
def discover_folder_structure(self, base_directory, name, override):
"""
- **Discover files and folders structure for OSM descriptors given a base_directory and name**
+ **Discover files and folders structure for SOL004/SOL007 descriptors given a base_directory and name**
:params:
- base_directory: is the location of the package to be created
files_folders = {
"folders": [
("{}_ns".format(prefix), "ns"),
- ("{}_ns/icons".format(prefix), "ns"),
- ("{}_ns/charms".format(prefix), "ns"),
+ ("{}_ns/Licenses".format(prefix), "ns"),
+ ("{}_ns/Files/icons".format(prefix), "ns"),
+ ("{}_ns/Scripts/charms".format(prefix), "ns"),
("{}_vnf".format(name), "vnf"),
- ("{}_vnf/charms".format(prefix), "vnf"),
- ("{}_vnf/cloud_init".format(prefix), "vnf"),
- ("{}_vnf/images".format(prefix), "vnf"),
- ("{}_vnf/icons".format(prefix), "vnf"),
- ("{}_vnf/scripts".format(prefix), "vnf"),
+ ("{}_vnf/Licenses".format(prefix), "vnf"),
+ ("{}_vnf/Scripts/charms".format(prefix), "vnf"),
+ ("{}_vnf/Scripts/cloud_init".format(prefix), "vnf"),
+ ("{}_vnf/Files/images".format(prefix), "vnf"),
+ ("{}_vnf/Files/icons".format(prefix), "vnf"),
+ ("{}_vnf/Scripts/scripts".format(prefix), "vnf"),
("{}_nst".format(prefix), "nst"),
("{}_nst/icons".format(prefix), "nst"),
],
("{}_ns/README.md".format(prefix), "ns", "readme"),
("{}_vnf/{}_vnfd.yaml".format(prefix, name), "vnf", "descriptor"),
(
- "{}_vnf/cloud_init/cloud-config.txt".format(prefix),
+ "{}_vnf/Scripts/cloud_init/cloud-config.txt".format(prefix),
"vnf",
"cloud_init",
),
# print("Missing files and folders: {}".format(missing_files_folders))
return missing_files_folders
- def charm_build(self, charms_folder, build_name):
+ def charm_build(self, charms_folder, build_name, sol004_007=True):
"""
Build the charms inside the package.
params: package_folder is the name of the folder where is the charms to compile.
build_name is the name of the layer or interface
"""
self._logger.debug("")
- os.environ["JUJU_REPOSITORY"] = "{}/charms".format(charms_folder)
+
+ if sol004_007:
+ os.environ["JUJU_REPOSITORY"] = "{}/Scripts/charms".format(charms_folder)
+ else:
+ os.environ["JUJU_REPOSITORY"] = "{}/charms".format(charms_folder)
+
os.environ["CHARM_LAYERS_DIR"] = "{}/layers".format(
os.environ["JUJU_REPOSITORY"]
)
os.environ["CHARM_INTERFACES_DIR"] = "{}/interfaces".format(
os.environ["JUJU_REPOSITORY"]
)
- os.environ["CHARM_BUILD_DIR"] = "{}/charms/builds".format(charms_folder)
+
+ if sol004_007:
+ os.environ["CHARM_BUILD_DIR"] = "{}/Scripts/charms/builds".format(
+ charms_folder
+ )
+ else:
+ os.environ["CHARM_BUILD_DIR"] = "{}/charms/builds".format(charms_folder)
+
if not os.path.exists(os.environ["CHARM_BUILD_DIR"]):
os.makedirs(os.environ["CHARM_BUILD_DIR"])
src_folder = "{}/{}".format(os.environ["CHARM_LAYERS_DIR"], build_name)
build_name is the name of the layer or interface
"""
self._logger.debug("Building charm {}".format(charm_name))
- src_folder = f"{package_folder}/charms/ops/{charm_name}"
+ src_folder = f"{package_folder}/Scripts/charms/ops/{charm_name}"
current_directory = os.getcwd()
os.chdir(src_folder)
try:
finally:
os.chdir(current_directory)
+ def build_compressed_file(self, package_folder, charm_list=None, sol004_007=True):
+ if sol004_007:
+ return self.build_zipfile(package_folder, charm_list)
+ else:
+ return self.build_tarfile(package_folder, charm_list)
+
+ def build_zipfile(self, package_folder, charm_list=None):
+ """
+ Creates a zip file given a package_folder
+ params: package_folder is the name of the folder to be packaged
+ returns: .zip name
+ """
+ self._logger.debug("")
+ cwd = None
+ try:
+ directory_name, package_name = self.create_temp_dir_sol004_007(
+ package_folder, charm_list
+ )
+ cwd = os.getcwd()
+ os.chdir(directory_name)
+ package_type = package_handling.get_package_type(package_folder)
+ print(package_type)
+
+ if (
+ package_handling.SOL007 == package_type
+ or package_handling.SOL007_TOSCA == package_type
+ ):
+ the_package = SOL007Package(package_folder)
+ elif (
+ package_handling.SOL004 == package_type
+ or package_handling.SOL004_TOSCA == package_type
+ ):
+ the_package = SOL004Package(package_folder)
+
+ the_package.create_or_update_metadata_file()
+
+ the_zip_package = shutil.make_archive(
+ os.path.join(cwd, package_name),
+ "zip",
+ os.path.join(directory_name, package_name),
+ )
+
+ print("Package created: {}".format(the_zip_package))
+
+ return the_zip_package
+
+ except Exception as exc:
+ raise ClientException(
+ "failure during build of zip file (create temp dir, calculate checksum, "
+ "zip file): {}".format(exc)
+ )
+ finally:
+ if cwd:
+ os.chdir(cwd)
+ shutil.rmtree(os.path.join(package_folder, "tmp"))
+
def build_tarfile(self, package_folder, charm_list=None):
"""
Creates a .tar.gz file given a package_folder
self._logger.debug(
"Copying tree: {} -> {}".format(s_charm, d_temp)
)
- shutil.copytree(
- s_charm, d_temp, symlinks=True, ignore=ignore
- )
+ if os.path.isdir(s_charm):
+ shutil.copytree(
+ s_charm, d_temp, symlinks=True, ignore=ignore
+ )
+ else:
+ shutil.copy2(s_charm, d_temp)
self._logger.debug("DONE")
else:
self._logger.debug("Copying tree: {} -> {}".format(s, d))
self._logger.debug("DONE")
return directory_name, package_name
+ def copy_tree(self, s, d, ignore):
+ self._logger.debug("Copying tree: {} -> {}".format(s, d))
+ shutil.copytree(s, d, symlinks=True, ignore=ignore)
+ self._logger.debug("DONE")
+
+ def create_temp_dir_sol004_007(self, package_folder, charm_list=None):
+ """
+ Method to create a temporary folder where we can move the files in package_folder
+ """
+ self._logger.debug("")
+ ignore_patterns = ".gitignore"
+ ignore = shutil.ignore_patterns(ignore_patterns)
+ directory_name = os.path.abspath(package_folder)
+ package_name = os.path.basename(directory_name)
+ directory_name += "/tmp"
+ os.makedirs("{}/{}".format(directory_name, package_name), exist_ok=True)
+ self._logger.debug("Makedirs DONE: {}/{}".format(directory_name, package_name))
+ for item in os.listdir(package_folder):
+ self._logger.debug("Item: {}".format(item))
+ if item != "tmp":
+ s = os.path.join(package_folder, item)
+ d = os.path.join(os.path.join(directory_name, package_name), item)
+ if os.path.isdir(s):
+ if item == "Scripts":
+ os.makedirs(d, exist_ok=True)
+ scripts_folder = s
+ for script_item in os.listdir(scripts_folder):
+ scripts_destination_folder = os.path.join(d, script_item)
+ if script_item == "charms":
+ s_builds = os.path.join(
+ scripts_folder, script_item, "builds"
+ )
+ for charm in charm_list:
+ self._logger.debug("Copying charm {}".format(charm))
+ if charm in os.listdir(
+ os.path.join(scripts_folder, script_item)
+ ):
+ s_charm = os.path.join(
+ scripts_folder, script_item, charm
+ )
+ elif charm in os.listdir(s_builds):
+ s_charm = os.path.join(s_builds, charm)
+ else:
+ raise ClientException(
+ "The charm {} referenced in the descriptor file "
+ "could not be found in {}/charms or in {}/charms/builds".format(
+ charm, package_folder, package_folder
+ )
+ )
+ d_temp = os.path.join(
+ scripts_destination_folder, charm
+ )
+ self.copy_tree(s_charm, d_temp, ignore)
+ else:
+ self.copy_tree(
+ os.path.join(scripts_folder, script_item),
+ scripts_destination_folder,
+ ignore,
+ )
+ else:
+ self.copy_tree(s, d, ignore)
+ else:
+ if item in ignore_patterns:
+ continue
+ self._logger.debug("Copying file: {} -> {}".format(s, d))
+ shutil.copy2(s, d)
+ self._logger.debug("DONE")
+ return directory_name, package_name
+
def charms_search(self, descriptor_file, desc_type):
self._logger.debug(
"descriptor_file: {}, desc_type: {}".format(descriptor_file, desc_type)