6 # Licensed under the Apache License, Version 2.0 (the "License"); you may
7 # not use this file except in compliance with the License. You may obtain
8 # a copy of the License at
10 # http://www.apache.org/licenses/LICENSE-2.0
12 # Unless required by applicable law or agreed to in writing, software
13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15 # License for the specific language governing permissions and limitations
27 from jinja2
import Environment
, PackageLoader
28 from osm_im
.validation
import Validation
as validation_im
29 from osm_im
.validation
import ValidationException
30 from osm_im
import im_translation
31 from osmclient
.common
.exceptions
import ClientException
35 class PackageTool(object):
36 def __init__(self
, client
=None):
38 self
._logger
= logging
.getLogger("osmclient")
39 self
._validator
= validation_im()
60 **Create a package descriptor**
63 - package_type: [vnf, ns, nst]
64 - base directory: path of destination folder
65 - package_name: is the name of the package to be created
66 - image: specify the image of the vdu
67 - vcpu: number of virtual cpus of the vdu
68 - memory: amount of memory in MB pf the vdu
69 - storage: amount of storage in GB of the vdu
70 - interfaces: number of interfaces besides management interface
71 - vendor: vendor name of the vnf/ns
72 - detailed: include all possible values for NSD, VNFD, NST
73 - netslice_subnets: number of netslice_subnets for the NST
74 - netslice_vlds: number of virtual link descriptors for the NST
75 - old: flag to create a package using the IM of OSM<9
79 self
._logger
.debug("")
80 # print("location: {}".format(osmclient.__path__))
81 file_loader
= PackageLoader("osmclient")
82 env
= Environment(loader
=file_loader
)
83 if package_type
== "ns":
84 template
= env
.get_template("nsd.yaml.j2" if not old
else "nsd_old.yaml.j2")
90 "interfaces": interfaces
,
93 elif package_type
== "vnf":
94 template
= env
.get_template(
95 "vnfd.yaml.j2" if not old
else "vnfd_old.yaml.j2"
102 "interfaces": interfaces
,
107 "detailed": detailed
,
109 elif package_type
== "nst":
110 # TODO: repo-index did not support nst in OSM<9, no changes in template
111 template
= env
.get_template("nst.yaml.j2")
113 "name": package_name
,
115 "interfaces": interfaces
,
116 "netslice_subnets": netslice_subnets
,
117 "netslice_vlds": netslice_vlds
,
118 "detailed": detailed
,
121 raise ClientException(
122 "Wrong descriptor type {}. Options: ns, vnf, nst".format(package_type
)
125 self
._logger
.debug("To be rendered: {}".format(content
))
126 output
= template
.render(content
)
127 self
._logger
.debug(output
)
129 structure
= self
.discover_folder_structure(
130 base_directory
, package_name
, override
132 if structure
.get("folders"):
133 self
.create_folders(structure
["folders"], package_type
)
134 if structure
.get("files"):
135 self
.create_files(structure
["files"], output
, package_type
)
138 def validate(self
, base_directory
, recursive
=True, old_format
=False):
140 **Validate OSM Descriptors given a path**
143 - base_directory is the root path for all descriptors
145 :return: List of dict of validated descriptors. keys: type, path, valid, error
147 self
._logger
.debug("")
150 descriptors_paths
= [
151 f
for f
in glob
.glob(base_directory
+ "/**/*.yaml", recursive
=recursive
)
154 descriptors_paths
= [
155 f
for f
in glob
.glob(base_directory
+ "/*.yaml", recursive
=recursive
)
157 self
._logger
.info("Base directory: {}".format(base_directory
))
159 "{} Descriptors found to validate".format(len(descriptors_paths
))
161 for desc_path
in descriptors_paths
:
162 with
open(desc_path
) as descriptor_file
:
163 descriptor_data
= descriptor_file
.read()
166 # TODO: refactor validation_im.yaml_validation to @staticmethod
167 desc_type
, descriptor_data
= validation_im
.yaml_validation(
168 self
, descriptor_data
170 self
._logger
.debug(f
"Validate {desc_type} {descriptor_data}")
172 if desc_type
== "vnfd" or desc_type
== "nsd":
174 "OSM descriptor '{}' written in an unsupported format. Please update to ETSI SOL006 format".format(
178 self
._logger
.warning(
179 "Package validation skipped. It can still be done with 'osm package-validate --old'"
181 self
._logger
.warning(
182 "Package build can still be done with 'osm package-build --skip-validation'"
184 raise Exception("Not SOL006 format")
185 validation_im
.pyangbind_validation(self
, desc_type
, descriptor_data
)
187 {"type": desc_type
, "path": desc_path
, "valid": "OK", "error": "-"}
189 except Exception as e
:
190 self
._logger
.error(f
"Validation error: {e}", exc_info
=True)
199 self
._logger
.debug(table
[-1])
202 def translate(self
, base_directory
, recursive
=True, dryrun
=False):
204 **Translate OSM Packages given a path**
207 - base_directory is the root path for all packages
209 :return: List of dict of translated packages. keys: current type, new type, path, valid, translated, error
211 self
._logger
.debug("")
214 descriptors_paths
= [
215 f
for f
in glob
.glob(base_directory
+ "/**/*.yaml", recursive
=recursive
)
218 descriptors_paths
= [
219 f
for f
in glob
.glob(base_directory
+ "/*.yaml", recursive
=recursive
)
221 print("Base directory: {}".format(base_directory
))
222 print("{} Descriptors found to validate".format(len(descriptors_paths
)))
223 for desc_path
in descriptors_paths
:
224 with
open(desc_path
) as descriptor_file
:
225 descriptor_data
= descriptor_file
.read()
228 desc_type
, descriptor_data
= validation_im
.yaml_validation(
229 self
, descriptor_data
231 self
._logger
.debug("desc_type: {}".format(desc_type
))
232 self
._logger
.debug("descriptor_data:\n{}".format(descriptor_data
))
233 self
._validator
.pyangbind_validation(desc_type
, descriptor_data
)
234 if not (desc_type
== "vnfd" or desc_type
== "nsd"):
237 "current type": desc_type
,
238 "new type": desc_type
,
246 new_desc_type
= desc_type
248 sol006_model
= yaml
.safe_dump(
249 im_translation
.translate_im_model_to_sol006(
253 default_flow_style
=False,
258 ) = self
._validator
.yaml_validation(sol006_model
)
259 self
._validator
.pyangbind_validation(
260 new_desc_type
, new_descriptor_data
263 with
open(desc_path
, "w") as descriptor_file
:
264 descriptor_file
.write(sol006_model
)
267 "current type": desc_type
,
268 "new type": new_desc_type
,
275 except ValidationException
as ve2
:
278 "current type": desc_type
,
279 "new type": new_desc_type
,
282 "translated": "ERROR",
283 "error": "Error in the post-validation: {}".format(
288 except Exception as e2
:
291 "current type": desc_type
,
292 "new type": new_desc_type
,
295 "translated": "ERROR",
296 "error": "Error in the translation: {}".format(str(e2
)),
299 except ValidationException
as ve
:
302 "current type": desc_type
,
307 "error": "Error in the pre-validation: {}".format(str(ve
)),
310 except Exception as e
:
313 "current type": desc_type
,
323 def descriptor_translate(self
, descriptor_file
):
325 **Translate input descriptor file from Rel EIGHT OSM to SOL006**
328 - base_directory is the root path for all packages
330 :return: YAML descriptor in the new format
332 self
._logger
.debug("")
333 with
open(descriptor_file
, "r") as df
:
334 im_model
= yaml
.safe_load(df
.read())
335 sol006_model
= im_translation
.translate_im_model_to_sol006(im_model
)
336 return yaml
.safe_dump(sol006_model
, indent
=4, default_flow_style
=False)
338 def build(self
, package_folder
, skip_validation
=False, skip_charm_build
=False):
340 **Creates a .tar.gz file given a package_folder**
343 - package_folder: is the name of the folder to be packaged
344 - skip_validation: is the flag to validate or not the descriptors on the folder before build
346 :returns: message result for the build process
348 self
._logger
.debug("")
349 package_folder
= package_folder
.rstrip("/")
350 if not os
.path
.exists("{}".format(package_folder
)):
351 return "Fail, package is not in the specified path"
352 if not skip_validation
:
353 print("Validating package {}".format(package_folder
))
354 results
= self
.validate(package_folder
, recursive
=False)
356 for result
in results
:
357 if result
["valid"] != "OK":
358 raise ClientException(
359 "There was an error validating the file {} with error: {}".format(
360 result
["path"], result
["error"]
363 print("Validation OK")
365 raise ClientException(
366 "No descriptor file found in: {}".format(package_folder
)
368 charm_list
= self
.build_all_charms(package_folder
, skip_charm_build
)
369 return self
.build_tarfile(package_folder
, charm_list
)
371 def calculate_checksum(self
, package_folder
):
373 **Function to calculate the checksum given a folder**
376 - package_folder: is the folder where we have the files to calculate the checksum
379 self
._logger
.debug("")
382 for f
in glob
.glob(package_folder
+ "/**/*.*", recursive
=True)
385 with
open("{}/checksums.txt".format(package_folder
), "w+") as checksum
:
386 for file_item
in files
:
387 if "checksums.txt" in file_item
:
389 # from https://www.quickprogrammingtips.com/python/how-to-calculate-md5-hash-of-a-file-in-python.html
390 md5_hash
= hashlib
.md5()
391 with
open(file_item
, "rb") as f
:
392 # Read and update hash in chunks of 4K
393 for byte_block
in iter(lambda: f
.read(4096), b
""):
394 md5_hash
.update(byte_block
)
395 checksum
.write("{}\t{}\n".format(md5_hash
.hexdigest(), file_item
))
397 def create_folders(self
, folders
, package_type
):
399 **Create folder given a list of folders**
402 - folders: [List] list of folders paths to be created
403 - package_type: is the type of package to be created
406 self
._logger
.debug("")
407 for folder
in folders
:
409 # print("Folder {} == package_type {}".format(folder[1], package_type))
410 if folder
[1] == package_type
:
411 print("Creating folder:\t{}".format(folder
[0]))
412 os
.makedirs(folder
[0])
413 except FileExistsError
:
416 def save_file(self
, file_name
, file_body
):
418 **Create a file given a name and the content**
421 - file_name: is the name of the file with the relative route
422 - file_body: is the content of the file
425 self
._logger
.debug("")
426 print("Creating file: \t{}".format(file_name
))
428 with
open(file_name
, "w+") as f
:
430 except Exception as e
:
431 raise ClientException(e
)
433 def generate_readme(self
):
435 **Creates the README content**
437 :returns: readme content
439 self
._logger
.debug("")
440 return """# Descriptor created by OSM descriptor package generated\n\n**Created on {} **""".format(
441 time
.strftime("%m/%d/%Y, %H:%M:%S", time
.localtime())
444 def generate_cloud_init(self
):
446 **Creates the cloud-init content**
448 :returns: cloud-init content
450 self
._logger
.debug("")
451 return "---\n#cloud-config"
453 def create_files(self
, files
, file_content
, package_type
):
455 **Creates the files given the file list and type**
458 - files: is the list of files structure
459 - file_content: is the content of the descriptor rendered by the template
460 - package_type: is the type of package to filter the creation structure
464 self
._logger
.debug("")
465 for file_item
, file_package
, file_type
in files
:
466 if package_type
== file_package
:
467 if file_type
== "descriptor":
468 self
.save_file(file_item
, file_content
)
469 elif file_type
== "readme":
470 self
.save_file(file_item
, self
.generate_readme())
471 elif file_type
== "cloud_init":
472 self
.save_file(file_item
, self
.generate_cloud_init())
474 def check_files_folders(self
, path_list
, override
):
476 **Find files and folders missing given a directory structure {"folders": [], "files": []}**
479 - path_list: is the list of files and folders to be created
480 - override: is the flag used to indicate the creation of the list even if the file exist to override it
482 :return: Missing paths Dict
484 self
._logger
.debug("")
488 for folder
in path_list
.get("folders"):
489 if not os
.path
.exists(folder
[0]):
490 folders
.append(folder
)
491 missing_paths
["folders"] = folders
493 for file_item
in path_list
.get("files"):
494 if not os
.path
.exists(file_item
[0]) or override
is True:
495 files
.append(file_item
)
496 missing_paths
["files"] = files
500 def build_all_charms(self
, package_folder
, skip_charm_build
):
502 **Read the descriptor file, check that the charms referenced are in the folder and compiles them**
505 - packet_folder: is the location of the package
506 :return: Files and Folders not found. In case of override, it will return all file list
508 self
._logger
.debug("")
510 descriptor_file
= False
511 descriptors_paths
= [f
for f
in glob
.glob(package_folder
+ "/*.yaml")]
512 for file in descriptors_paths
:
513 if file.endswith("nfd.yaml"):
514 descriptor_file
= True
515 listCharms
= self
.charms_search(file, "vnf")
516 if file.endswith("nsd.yaml"):
517 descriptor_file
= True
518 listCharms
= self
.charms_search(file, "ns")
519 print("List of charms in the descriptor: {}".format(listCharms
))
520 if not descriptor_file
:
521 raise ClientException(
522 'Descriptor filename is not correct in: {}. It should end with "nfd.yaml" or "nsd.yaml"'.format(
526 if listCharms
and not skip_charm_build
:
527 for charmName
in listCharms
:
529 "{}/charms/layers/{}".format(package_folder
, charmName
)
532 "Building charm {}/charms/layers/{}".format(
533 package_folder
, charmName
536 self
.charm_build(package_folder
, charmName
)
537 print("Charm built: {}".format(charmName
))
539 "{}/charms/ops/{}".format(package_folder
, charmName
)
541 self
.charmcraft_build(package_folder
, charmName
)
543 if not os
.path
.isdir(
544 "{}/charms/{}".format(package_folder
, charmName
)
545 ) and not os
.path
.isfile(
546 "{}/charms/{}".format(package_folder
, charmName
)
548 raise ClientException(
549 "The charm: {} referenced in the descriptor file "
550 "is not present either in {}/charms or in {}/charms/layers".format(
551 charmName
, package_folder
, package_folder
554 self
._logger
.debug("Return list of charms: {}".format(listCharms
))
557 def discover_folder_structure(self
, base_directory
, name
, override
):
559 **Discover files and folders structure for OSM descriptors given a base_directory and name**
562 - base_directory: is the location of the package to be created
563 - name: is the name of the package
564 - override: is the flag used to indicate the creation of the list even if the file exist to override it
565 :return: Files and Folders not found. In case of override, it will return all file list
567 self
._logger
.debug("")
568 prefix
= "{}/{}".format(base_directory
, name
)
571 ("{}_ns".format(prefix
), "ns"),
572 ("{}_ns/icons".format(prefix
), "ns"),
573 ("{}_ns/charms".format(prefix
), "ns"),
574 ("{}_vnf".format(name
), "vnf"),
575 ("{}_vnf/charms".format(prefix
), "vnf"),
576 ("{}_vnf/cloud_init".format(prefix
), "vnf"),
577 ("{}_vnf/images".format(prefix
), "vnf"),
578 ("{}_vnf/icons".format(prefix
), "vnf"),
579 ("{}_vnf/scripts".format(prefix
), "vnf"),
580 ("{}_nst".format(prefix
), "nst"),
581 ("{}_nst/icons".format(prefix
), "nst"),
584 ("{}_ns/{}_nsd.yaml".format(prefix
, name
), "ns", "descriptor"),
585 ("{}_ns/README.md".format(prefix
), "ns", "readme"),
586 ("{}_vnf/{}_vnfd.yaml".format(prefix
, name
), "vnf", "descriptor"),
588 "{}_vnf/cloud_init/cloud-config.txt".format(prefix
),
592 ("{}_vnf/README.md".format(prefix
), "vnf", "readme"),
593 ("{}_nst/{}_nst.yaml".format(prefix
, name
), "nst", "descriptor"),
594 ("{}_nst/README.md".format(prefix
), "nst", "readme"),
597 missing_files_folders
= self
.check_files_folders(files_folders
, override
)
598 # print("Missing files and folders: {}".format(missing_files_folders))
599 return missing_files_folders
601 def charm_build(self
, charms_folder
, build_name
):
603 Build the charms inside the package.
604 params: package_folder is the name of the folder where is the charms to compile.
605 build_name is the name of the layer or interface
607 self
._logger
.debug("")
608 os
.environ
["JUJU_REPOSITORY"] = "{}/charms".format(charms_folder
)
609 os
.environ
["CHARM_LAYERS_DIR"] = "{}/layers".format(
610 os
.environ
["JUJU_REPOSITORY"]
612 os
.environ
["CHARM_INTERFACES_DIR"] = "{}/interfaces".format(
613 os
.environ
["JUJU_REPOSITORY"]
615 os
.environ
["CHARM_BUILD_DIR"] = "{}/charms/builds".format(charms_folder
)
616 if not os
.path
.exists(os
.environ
["CHARM_BUILD_DIR"]):
617 os
.makedirs(os
.environ
["CHARM_BUILD_DIR"])
618 src_folder
= "{}/{}".format(os
.environ
["CHARM_LAYERS_DIR"], build_name
)
619 result
= subprocess
.run(["charm", "build", "{}".format(src_folder
)])
620 if result
.returncode
== 1:
621 raise ClientException("failed to build the charm: {}".format(src_folder
))
622 self
._logger
.verbose("charm {} built".format(src_folder
))
624 def charmcraft_build(self
, package_folder
, charm_name
):
626 Build the charms inside the package (new operator framework charms)
627 params: package_folder is the name of the folder where is the charms to compile.
628 build_name is the name of the layer or interface
630 self
._logger
.debug("Building charm {}".format(charm_name
))
631 src_folder
= f
"{package_folder}/charms/ops/{charm_name}"
632 current_directory
= os
.getcwd()
635 result
= subprocess
.run(["charmcraft", "build"])
636 if result
.returncode
== 1:
637 raise ClientException(
638 "failed to build the charm: {}".format(src_folder
)
640 subprocess
.run(["rm", "-rf", f
"../../{charm_name}"])
641 subprocess
.run(["mv", "build", f
"../../{charm_name}"])
642 self
._logger
.verbose("charm {} built".format(src_folder
))
644 os
.chdir(current_directory
)
646 def build_tarfile(self
, package_folder
, charm_list
=None):
648 Creates a .tar.gz file given a package_folder
649 params: package_folder is the name of the folder to be packaged
650 returns: .tar.gz name
652 self
._logger
.debug("")
655 directory_name
, package_name
= self
.create_temp_dir(
656 package_folder
, charm_list
659 os
.chdir(directory_name
)
660 self
.calculate_checksum(package_name
)
661 with tarfile
.open("{}.tar.gz".format(package_name
), mode
="w:gz") as archive
:
662 print("Adding File: {}".format(package_name
))
663 archive
.add("{}".format(package_name
), recursive
=True)
664 # return "Created {}.tar.gz".format(package_folder)
665 # self.build("{}".format(os.path.basename(package_folder)))
668 created_package
= "{}/{}.tar.gz".format(
669 os
.path
.dirname(package_folder
) or ".", package_name
672 "{}/{}.tar.gz".format(directory_name
, package_name
), created_package
675 "{}/{}/checksums.txt".format(directory_name
, package_name
),
676 "{}/checksums.txt".format(package_folder
),
678 print("Package created: {}".format(created_package
))
679 return created_package
680 except Exception as exc
:
681 raise ClientException(
682 "failure during build of targz file (create temp dir, calculate checksum, "
683 "tar.gz file): {}".format(exc
)
688 shutil
.rmtree(os
.path
.join(package_folder
, "tmp"))
690 def create_temp_dir(self
, package_folder
, charm_list
=None):
692 Method to create a temporary folder where we can move the files in package_folder
694 self
._logger
.debug("")
695 ignore_patterns
= ".gitignore"
696 ignore
= shutil
.ignore_patterns(ignore_patterns
)
697 directory_name
= os
.path
.abspath(package_folder
)
698 package_name
= os
.path
.basename(directory_name
)
699 directory_name
+= "/tmp"
700 os
.makedirs("{}/{}".format(directory_name
, package_name
), exist_ok
=True)
701 self
._logger
.debug("Makedirs DONE: {}/{}".format(directory_name
, package_name
))
702 for item
in os
.listdir(package_folder
):
703 self
._logger
.debug("Item: {}".format(item
))
705 s
= os
.path
.join(package_folder
, item
)
706 d
= os
.path
.join(os
.path
.join(directory_name
, package_name
), item
)
709 os
.makedirs(d
, exist_ok
=True)
710 s_builds
= os
.path
.join(s
, "builds")
711 for charm
in charm_list
:
712 self
._logger
.debug("Copying charm {}".format(charm
))
713 if charm
in os
.listdir(s
):
714 s_charm
= os
.path
.join(s
, charm
)
715 elif charm
in os
.listdir(s_builds
):
716 s_charm
= os
.path
.join(s_builds
, charm
)
718 raise ClientException(
719 "The charm {} referenced in the descriptor file "
720 "could not be found in {}/charms or in {}/charms/builds".format(
721 charm
, package_folder
, package_folder
724 d_temp
= os
.path
.join(d
, charm
)
726 "Copying tree: {} -> {}".format(s_charm
, d_temp
)
728 if os
.path
.isdir(s_charm
):
730 s_charm
, d_temp
, symlinks
=True, ignore
=ignore
733 shutil
.copy2(s_charm
, d_temp
)
734 self
._logger
.debug("DONE")
736 self
._logger
.debug("Copying tree: {} -> {}".format(s
, d
))
737 shutil
.copytree(s
, d
, symlinks
=True, ignore
=ignore
)
738 self
._logger
.debug("DONE")
740 if item
in ignore_patterns
:
742 self
._logger
.debug("Copying file: {} -> {}".format(s
, d
))
744 self
._logger
.debug("DONE")
745 return directory_name
, package_name
747 def charms_search(self
, descriptor_file
, desc_type
):
749 "descriptor_file: {}, desc_type: {}".format(descriptor_file
, desc_type
)
751 with
open("{}".format(descriptor_file
)) as yaml_desc
:
752 descriptor_dict
= yaml
.safe_load(yaml_desc
)
753 # self._logger.debug("\n"+yaml.safe_dump(descriptor_dict, indent=4, default_flow_style=False))
758 "vnfd:vnfd-catalog" in descriptor_dict
759 or "vnfd-catalog" in descriptor_dict
764 "nsd:nsd-catalog" in descriptor_dict
765 or "nsd-catalog" in descriptor_dict
768 charms_list
= self
._charms
_search
_on
_osm
_im
_dict
(
769 descriptor_dict
, desc_type
772 if desc_type
== "ns":
773 get_charm_list
= self
._charms
_search
_on
_nsd
_sol
006_dict
774 elif desc_type
== "vnf":
775 get_charm_list
= self
._charms
_search
_on
_vnfd
_sol
006_dict
777 raise Exception("Bad descriptor type")
778 charms_list
= get_charm_list(descriptor_dict
)
781 def _charms_search_on_osm_im_dict(self
, osm_im_dict
, desc_type
):
782 self
._logger
.debug("")
784 for k1
, v1
in osm_im_dict
.items():
785 for k2
, v2
in v1
.items():
787 if "{}-configuration".format(desc_type
) in entry
:
788 vnf_config
= entry
["{}-configuration".format(desc_type
)]
789 for k3
, v3
in vnf_config
.items():
791 charms_list
.append((v3
["charm"]))
795 if "vdu-configuration" in vdu
:
796 for k4
, v4
in vdu
["vdu-configuration"].items():
798 charms_list
.append((v4
["charm"]))
801 def _charms_search_on_vnfd_sol006_dict(self
, sol006_dict
):
802 self
._logger
.debug("")
804 dfs
= sol006_dict
.get("vnfd", {}).get("df", [])
807 df
.get("lcm-operations-configuration", {})
808 .get("operate-vnf-op-config", {})
811 if day_1_2s
is not None:
812 for day_1_2
in day_1_2s
:
813 exec_env_list
= day_1_2
.get("execution-environment-list", [])
814 for exec_env
in exec_env_list
:
815 if "juju" in exec_env
and "charm" in exec_env
["juju"]:
816 charms_list
.append(exec_env
["juju"]["charm"])
819 def _charms_search_on_nsd_sol006_dict(self
, sol006_dict
):
820 self
._logger
.debug("")
822 nsd_list
= sol006_dict
.get("nsd", {}).get("nsd", [])
824 charm
= nsd
.get("ns-configuration", {}).get("juju", {}).get("charm")
826 charms_list
.append(charm
)