# under the License.
import glob
-import hashlib
import logging
import os
import shutil
import subprocess
import tarfile
import time
-
from jinja2 import Environment, PackageLoader
from osm_im.validation import Validation as validation_im
from osm_im.validation import ValidationException
from osm_im import im_translation
+from osmclient.common import package_handling as package_handling
from osmclient.common.exceptions import ClientException
+from osmclient.common import utils
+from .sol004_package import SOL004Package
+from .sol007_package import SOL007Package
import yaml
self._logger.debug("")
# print("location: {}".format(osmclient.__path__))
file_loader = PackageLoader("osmclient")
- env = Environment(loader=file_loader)
+ env = Environment(loader=file_loader, autoescape=True)
if package_type == "ns":
template = env.get_template("nsd.yaml.j2" if not old else "nsd_old.yaml.j2")
content = {
"detailed": detailed,
}
elif package_type == "vnf":
- template = env.get_template("vnfd.yaml.j2" if not old else "vnfd_old.yaml.j2")
+ template = env.get_template(
+ "vnfd.yaml.j2" if not old else "vnfd_old.yaml.j2"
+ )
content = {
"name": package_name,
"vendor": vendor,
f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)
]
self._logger.info("Base directory: {}".format(base_directory))
- self._logger.info("{} Descriptors found to validate".format(len(descriptors_paths)))
+ self._logger.info(
+ "{} Descriptors found to validate".format(len(descriptors_paths))
+ )
for desc_path in descriptors_paths:
with open(desc_path) as descriptor_file:
descriptor_data = descriptor_file.read()
raise ClientException(
"No descriptor file found in: {}".format(package_folder)
)
- charm_list = self.build_all_charms(package_folder, skip_charm_build)
- return self.build_tarfile(package_folder, charm_list)
+
+ is_sol004_007 = (
+ package_handling.get_package_type(package_folder)
+ != package_handling.OSM_OLD
+ )
+
+ charm_list = self.build_all_charms(
+ package_folder, skip_charm_build, is_sol004_007
+ )
+ return self.build_compressed_file(package_folder, charm_list, is_sol004_007)
def calculate_checksum(self, package_folder):
"""
for file_item in files:
if "checksums.txt" in file_item:
continue
- # from https://www.quickprogrammingtips.com/python/how-to-calculate-md5-hash-of-a-file-in-python.html
- md5_hash = hashlib.md5()
- with open(file_item, "rb") as f:
- # Read and update hash in chunks of 4K
- for byte_block in iter(lambda: f.read(4096), b""):
- md5_hash.update(byte_block)
- checksum.write("{}\t{}\n".format(md5_hash.hexdigest(), file_item))
+ checksum.write("{}\t{}\n".format(utils.md5(file_item), file_item))
def create_folders(self, folders, package_type):
"""
return missing_paths
- def build_all_charms(self, package_folder, skip_charm_build):
+ def build_all_charms(self, package_folder, skip_charm_build, sol004_007=True):
"""
**Read the descriptor file, check that the charms referenced are in the folder and compiles them**
self._logger.debug("")
charms_set = set()
descriptor_file = False
- descriptors_paths = [f for f in glob.glob(package_folder + "/*.yaml")]
+ package_type = package_handling.get_package_type(package_folder)
+ if sol004_007 and package_type.find("TOSCA") >= 0:
+ descriptors_paths = [
+ f for f in glob.glob(package_folder + "/Definitions/*.yaml")
+ ]
+ else:
+ descriptors_paths = [f for f in glob.glob(package_folder + "/*.yaml")]
for file in descriptors_paths:
if file.endswith("nfd.yaml"):
descriptor_file = True
if charms_set and not skip_charm_build:
for charmName in charms_set:
if os.path.isdir(
- "{}/charms/layers/{}".format(package_folder, charmName)
+ "{}/{}charms/layers/{}".format(
+ package_folder, "Scripts/" if sol004_007 else "", charmName
+ )
):
print(
- "Building charm {}/charms/layers/{}".format(
- package_folder, charmName
+ "Building charm {}/{}charms/layers/{}".format(
+ package_folder, "Scripts/" if sol004_007 else "", charmName
)
)
- self.charm_build(package_folder, charmName)
+ self.charm_build(package_folder, charmName, sol004_007)
print("Charm built: {}".format(charmName))
elif os.path.isdir(
- "{}/charms/ops/{}".format(package_folder, charmName)
+ "{}/{}charms/ops/{}".format(
+ package_folder, "Scripts/" if sol004_007 else "", charmName
+ )
):
self.charmcraft_build(package_folder, charmName)
else:
if not os.path.isdir(
- "{}/charms/{}".format(package_folder, charmName)
+ "{}/{}charms/{}".format(
+ package_folder, "Scripts/" if sol004_007 else "", charmName
+ )
+ ) and not os.path.isfile(
+ "{}/{}charms/{}".format(
+ package_folder, "Scripts/" if sol004_007 else "", charmName
+ )
):
raise ClientException(
"The charm: {} referenced in the descriptor file "
def discover_folder_structure(self, base_directory, name, override):
"""
- **Discover files and folders structure for OSM descriptors given a base_directory and name**
+ **Discover files and folders structure for SOL004/SOL007 descriptors given a base_directory and name**
:params:
- base_directory: is the location of the package to be created
files_folders = {
"folders": [
("{}_ns".format(prefix), "ns"),
- ("{}_ns/icons".format(prefix), "ns"),
- ("{}_ns/charms".format(prefix), "ns"),
+ ("{}_ns/Licenses".format(prefix), "ns"),
+ ("{}_ns/Files/icons".format(prefix), "ns"),
+ ("{}_ns/Scripts/charms".format(prefix), "ns"),
("{}_vnf".format(name), "vnf"),
- ("{}_vnf/charms".format(prefix), "vnf"),
- ("{}_vnf/cloud_init".format(prefix), "vnf"),
- ("{}_vnf/images".format(prefix), "vnf"),
- ("{}_vnf/icons".format(prefix), "vnf"),
- ("{}_vnf/scripts".format(prefix), "vnf"),
+ ("{}_vnf/Licenses".format(prefix), "vnf"),
+ ("{}_vnf/Scripts/charms".format(prefix), "vnf"),
+ ("{}_vnf/Scripts/cloud_init".format(prefix), "vnf"),
+ ("{}_vnf/Files/images".format(prefix), "vnf"),
+ ("{}_vnf/Files/icons".format(prefix), "vnf"),
+ ("{}_vnf/Scripts/scripts".format(prefix), "vnf"),
("{}_nst".format(prefix), "nst"),
("{}_nst/icons".format(prefix), "nst"),
],
("{}_ns/README.md".format(prefix), "ns", "readme"),
("{}_vnf/{}_vnfd.yaml".format(prefix, name), "vnf", "descriptor"),
(
- "{}_vnf/cloud_init/cloud-config.txt".format(prefix),
+ "{}_vnf/Scripts/cloud_init/cloud-config.txt".format(prefix),
"vnf",
"cloud_init",
),
# print("Missing files and folders: {}".format(missing_files_folders))
return missing_files_folders
- def charm_build(self, charms_folder, build_name):
+ def charm_build(self, charms_folder, build_name, sol004_007=True):
"""
Build the charms inside the package.
params: package_folder is the name of the folder where is the charms to compile.
build_name is the name of the layer or interface
"""
self._logger.debug("")
- os.environ["JUJU_REPOSITORY"] = "{}/charms".format(charms_folder)
+
+ if sol004_007:
+ os.environ["JUJU_REPOSITORY"] = "{}/Scripts/charms".format(charms_folder)
+ else:
+ os.environ["JUJU_REPOSITORY"] = "{}/charms".format(charms_folder)
+
os.environ["CHARM_LAYERS_DIR"] = "{}/layers".format(
os.environ["JUJU_REPOSITORY"]
)
os.environ["CHARM_INTERFACES_DIR"] = "{}/interfaces".format(
os.environ["JUJU_REPOSITORY"]
)
- os.environ["CHARM_BUILD_DIR"] = "{}/charms/builds".format(charms_folder)
+
+ if sol004_007:
+ os.environ["CHARM_BUILD_DIR"] = "{}/Scripts/charms/builds".format(
+ charms_folder
+ )
+ else:
+ os.environ["CHARM_BUILD_DIR"] = "{}/charms/builds".format(charms_folder)
+
if not os.path.exists(os.environ["CHARM_BUILD_DIR"]):
os.makedirs(os.environ["CHARM_BUILD_DIR"])
src_folder = "{}/{}".format(os.environ["CHARM_LAYERS_DIR"], build_name)
build_name is the name of the layer or interface
"""
self._logger.debug("Building charm {}".format(charm_name))
- src_folder = f"{package_folder}/charms/ops/{charm_name}"
+ src_folder = f"{package_folder}/Scripts/charms/ops/{charm_name}"
current_directory = os.getcwd()
os.chdir(src_folder)
try:
finally:
os.chdir(current_directory)
+ def build_compressed_file(self, package_folder, charm_list=None, sol004_007=True):
+ if sol004_007:
+ return self.build_zipfile(package_folder, charm_list)
+ else:
+ return self.build_tarfile(package_folder, charm_list)
+
+ def build_zipfile(self, package_folder, charm_list=None):
+ """
+ Creates a zip file given a package_folder
+ params: package_folder is the name of the folder to be packaged
+ returns: .zip name
+ """
+ self._logger.debug("")
+ cwd = None
+ try:
+ directory_name, package_name = self.create_temp_dir_sol004_007(
+ package_folder, charm_list
+ )
+ cwd = os.getcwd()
+ os.chdir(directory_name)
+ package_type = package_handling.get_package_type(package_folder)
+ print(package_type)
+
+ if (
+ package_handling.SOL007 == package_type
+ or package_handling.SOL007_TOSCA == package_type
+ ):
+ the_package = SOL007Package(package_folder)
+ elif (
+ package_handling.SOL004 == package_type
+ or package_handling.SOL004_TOSCA == package_type
+ ):
+ the_package = SOL004Package(package_folder)
+ else:
+ raise ClientException(f"Unknown package type: {package_type}")
+
+ the_package.create_or_update_metadata_file()
+
+ the_zip_package = shutil.make_archive(
+ os.path.join(cwd, package_name),
+ "zip",
+ os.path.join(directory_name, package_name),
+ )
+
+ print("Package created: {}".format(the_zip_package))
+
+ return the_zip_package
+
+ except Exception as exc:
+ raise ClientException(
+ "failure during build of zip file (create temp dir, calculate checksum, "
+ "zip file): {}".format(exc)
+ )
+ finally:
+ if cwd:
+ os.chdir(cwd)
+ shutil.rmtree(os.path.join(package_folder, "tmp"))
+
def build_tarfile(self, package_folder, charm_list=None):
"""
Creates a .tar.gz file given a package_folder
self._logger.debug(
"Copying tree: {} -> {}".format(s_charm, d_temp)
)
- shutil.copytree(
- s_charm, d_temp, symlinks=True, ignore=ignore
- )
+ if os.path.isdir(s_charm):
+ shutil.copytree(
+ s_charm, d_temp, symlinks=True, ignore=ignore
+ )
+ else:
+ shutil.copy2(s_charm, d_temp)
self._logger.debug("DONE")
else:
self._logger.debug("Copying tree: {} -> {}".format(s, d))
self._logger.debug("DONE")
return directory_name, package_name
+ def copy_tree(self, s, d, ignore):
+ self._logger.debug("Copying tree: {} -> {}".format(s, d))
+ shutil.copytree(s, d, symlinks=True, ignore=ignore)
+ self._logger.debug("DONE")
+
+ def create_temp_dir_sol004_007(self, package_folder, charm_list=None):
+ """
+ Method to create a temporary folder where we can move the files in package_folder
+ """
+ self._logger.debug("")
+ ignore_patterns = ".gitignore"
+ ignore = shutil.ignore_patterns(ignore_patterns)
+ directory_name = os.path.abspath(package_folder)
+ package_name = os.path.basename(directory_name)
+ directory_name += "/tmp"
+ os.makedirs("{}/{}".format(directory_name, package_name), exist_ok=True)
+ self._logger.debug("Makedirs DONE: {}/{}".format(directory_name, package_name))
+ for item in os.listdir(package_folder):
+ self._logger.debug("Item: {}".format(item))
+ if item != "tmp":
+ s = os.path.join(package_folder, item)
+ d = os.path.join(os.path.join(directory_name, package_name), item)
+ if os.path.isdir(s):
+ if item == "Scripts":
+ os.makedirs(d, exist_ok=True)
+ scripts_folder = s
+ for script_item in os.listdir(scripts_folder):
+ scripts_destination_folder = os.path.join(d, script_item)
+ if script_item == "charms":
+ s_builds = os.path.join(
+ scripts_folder, script_item, "builds"
+ )
+ for charm in charm_list:
+ self._logger.debug("Copying charm {}".format(charm))
+ if charm in os.listdir(
+ os.path.join(scripts_folder, script_item)
+ ):
+ s_charm = os.path.join(
+ scripts_folder, script_item, charm
+ )
+ elif charm in os.listdir(s_builds):
+ s_charm = os.path.join(s_builds, charm)
+ else:
+ raise ClientException(
+ "The charm {} referenced in the descriptor file "
+ "could not be found in {}/charms or in {}/charms/builds".format(
+ charm, package_folder, package_folder
+ )
+ )
+ d_temp = os.path.join(
+ scripts_destination_folder, charm
+ )
+ self.copy_tree(s_charm, d_temp, ignore)
+ else:
+ self.copy_tree(
+ os.path.join(scripts_folder, script_item),
+ scripts_destination_folder,
+ ignore,
+ )
+ else:
+ self.copy_tree(s, d, ignore)
+ else:
+ if item in ignore_patterns:
+ continue
+ self._logger.debug("Copying file: {} -> {}".format(s, d))
+ shutil.copy2(s, d)
+ self._logger.debug("DONE")
+ return directory_name, package_name
+
def charms_search(self, descriptor_file, desc_type):
self._logger.debug(
"descriptor_file: {}, desc_type: {}".format(descriptor_file, desc_type)
def _charms_search_on_osm_im_dict(self, osm_im_dict, desc_type):
self._logger.debug("")
- charms_set = []
+ charms_set = set()
for k1, v1 in osm_im_dict.items():
for k2, v2 in v1.items():
for entry in v2: