aa31ed8536551f9e9f626e7778b81fabdffd04cb
6 # Licensed under the Apache License, Version 2.0 (the "License"); you may
7 # not use this file except in compliance with the License. You may obtain
8 # a copy of the License at
10 # http://www.apache.org/licenses/LICENSE-2.0
12 # Unless required by applicable law or agreed to in writing, software
13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15 # License for the specific language governing permissions and limitations
27 from jinja2
import Environment
, PackageLoader
28 from osm_im
.validation
import Validation
as validation_im
29 from osm_im
.validation
import ValidationException
30 from osm_im
import im_translation
31 from osmclient
.common
.exceptions
import ClientException
35 class PackageTool(object):
36 def __init__(self
, client
=None):
38 self
._logger
= logging
.getLogger('osmclient')
39 self
._validator
= validation_im()
41 def create(self
, package_type
, base_directory
, package_name
, override
, image
, vdus
, vcpu
, memory
, storage
,
42 interfaces
, vendor
, detailed
, netslice_subnets
, netslice_vlds
, old
):
44 **Create a package descriptor**
47 - package_type: [vnf, ns, nst]
48 - base directory: path of destination folder
49 - package_name: is the name of the package to be created
50 - image: specify the image of the vdu
51 - vcpu: number of virtual cpus of the vdu
52 - memory: amount of memory in MB pf the vdu
53 - storage: amount of storage in GB of the vdu
54 - interfaces: number of interfaces besides management interface
55 - vendor: vendor name of the vnf/ns
56 - detailed: include all possible values for NSD, VNFD, NST
57 - netslice_subnets: number of netslice_subnets for the NST
58 - netslice_vlds: number of virtual link descriptors for the NST
59 - old: flag to create a package using the IM of OSM<9
63 self
._logger
.debug("")
64 # print("location: {}".format(osmclient.__path__))
65 file_loader
= PackageLoader("osmclient")
66 env
= Environment(loader
=file_loader
)
67 if package_type
== 'ns':
68 template
= env
.get_template("nsd.yaml.j2" if not old
else "nsd_old.yaml.j2")
69 content
= {"name": package_name
, "vendor": vendor
, "vdus": vdus
, "clean": False, "interfaces": interfaces
,
71 elif package_type
== 'vnf':
72 template
= env
.get_template("vnfd.yaml.j2" if not old
else "vnfd_old.yaml.j2")
73 content
= {"name": package_name
, "vendor": vendor
, "vdus": vdus
, "clean": False, "interfaces": interfaces
,
74 "image": image
, "vcpu": vcpu
, "memory": memory
, "storage": storage
, "detailed": detailed
}
75 elif package_type
== 'nst':
76 template
= env
.get_template('nst.yaml.j2')
77 content
= {"name": package_name
, "vendor": vendor
, "interfaces": interfaces
,
78 "netslice_subnets": netslice_subnets
, "netslice_vlds": netslice_vlds
, "detailed": detailed
}
80 raise ClientException("Wrong descriptor type {}. Options: ns, vnf, nst".format(package_type
))
82 # print("To be rendered: {}".format(content))
83 output
= template
.render(content
)
86 structure
= self
.discover_folder_structure(base_directory
, package_name
, override
)
87 if structure
.get("folders"):
88 self
.create_folders(structure
["folders"], package_type
)
89 if structure
.get("files"):
90 self
.create_files(structure
["files"], output
, package_type
)
93 def validate(self
, base_directory
, recursive
=True, old_format
=False):
95 **Validate OSM Descriptors given a path**
98 - base_directory is the root path for all descriptors
100 :return: List of dict of validated descriptors. keys: type, path, valid, error
102 self
._logger
.debug("")
105 descriptors_paths
= [f
for f
in glob
.glob(base_directory
+ "/**/*.yaml", recursive
=recursive
)]
107 descriptors_paths
= [f
for f
in glob
.glob(base_directory
+ "/*.yaml", recursive
=recursive
)]
108 print("Base directory: {}".format(base_directory
))
109 print("{} Descriptors found to validate".format(len(descriptors_paths
)))
110 for desc_path
in descriptors_paths
:
111 with
open(desc_path
) as descriptor_file
:
112 descriptor_data
= descriptor_file
.read()
115 desc_type
, descriptor_data
= validation_im
.yaml_validation(self
, descriptor_data
)
117 if ( desc_type
=="vnfd" or desc_type
=="nsd" ):
118 print("OSM descriptor '{}' written in an unsupported format. Please update to ETSI SOL006 format".format(desc_path
))
119 print("Package validation skipped. It can still be done with 'osm package-validate --old'")
120 print("Package build can still be done with 'osm package-build --skip-validation'")
121 raise Exception("Not SOL006 format")
122 validation_im
.pyangbind_validation(self
, desc_type
, descriptor_data
)
123 table
.append({"type": desc_type
, "path": desc_path
, "valid": "OK", "error": "-"})
124 except Exception as e
:
125 table
.append({"type": desc_type
, "path": desc_path
, "valid": "ERROR", "error": str(e
)})
128 def translate(self
, base_directory
, recursive
=True, dryrun
=False):
130 **Translate OSM Packages given a path**
133 - base_directory is the root path for all packages
135 :return: List of dict of translated packages. keys: current type, new type, path, valid, translated, error
137 self
._logger
.debug("")
140 descriptors_paths
= [f
for f
in glob
.glob(base_directory
+ "/**/*.yaml", recursive
=recursive
)]
142 descriptors_paths
= [f
for f
in glob
.glob(base_directory
+ "/*.yaml", recursive
=recursive
)]
143 print("Base directory: {}".format(base_directory
))
144 print("{} Descriptors found to validate".format(len(descriptors_paths
)))
145 for desc_path
in descriptors_paths
:
146 with
open(desc_path
) as descriptor_file
:
147 descriptor_data
= descriptor_file
.read()
150 desc_type
, descriptor_data
= validation_im
.yaml_validation(self
, descriptor_data
)
151 self
._logger
.debug("desc_type: {}".format(desc_type
))
152 self
._logger
.debug("descriptor_data:\n{}".format(descriptor_data
))
153 self
._validator
.pyangbind_validation(desc_type
, descriptor_data
)
154 if not ( desc_type
=="vnfd" or desc_type
=="nsd" ):
155 table
.append({"current type": desc_type
, "new type": desc_type
, "path": desc_path
, "valid": "OK", "translated": "N/A", "error": "-"})
157 new_desc_type
= desc_type
159 sol006_model
= yaml
.safe_dump(im_translation
.translate_im_model_to_sol006(descriptor_data
), indent
=4, default_flow_style
=False)
160 new_desc_type
, new_descriptor_data
= self
._validator
.yaml_validation(sol006_model
)
161 self
._validator
.pyangbind_validation(new_desc_type
, new_descriptor_data
)
163 with
open(desc_path
, 'w') as descriptor_file
:
164 descriptor_file
.write(sol006_model
)
165 table
.append({"current type": desc_type
, "new type": new_desc_type
, "path": desc_path
, "valid": "OK", "translated": "OK", "error": "-"})
166 except ValidationException
as ve2
:
167 table
.append({"current type": desc_type
, "new type": new_desc_type
, "path": desc_path
, "valid": "OK", "translated": "ERROR", "error": "Error in the post-validation: {}".format(str(ve2
))})
168 except Exception as e2
:
169 table
.append({"current type": desc_type
, "new type": new_desc_type
, "path": desc_path
, "valid": "OK", "translated": "ERROR", "error": "Error in the translation: {}".format(str(e2
))})
170 except ValidationException
as ve
:
171 table
.append({"current type": desc_type
, "new type": "N/A", "path": desc_path
, "valid": "ERROR", "translated": "N/A", "error": "Error in the pre-validation: {}".format(str(ve
))})
172 except Exception as e
:
173 table
.append({"current type": desc_type
, "new type": "N/A", "path": desc_path
, "valid": "ERROR", "translated": "N/A", "error": str(e
)})
176 def descriptor_translate(self
, descriptor_file
):
178 **Translate input descriptor file from Rel EIGHT OSM to SOL006**
181 - base_directory is the root path for all packages
183 :return: YAML descriptor in the new format
185 self
._logger
.debug("")
186 with
open(descriptor_file
, 'r') as df
:
187 im_model
= yaml
.safe_load(df
.read())
188 sol006_model
= im_translation
.translate_im_model_to_sol006(im_model
)
189 return yaml
.safe_dump(sol006_model
, indent
=4, default_flow_style
=False)
191 def build(self
, package_folder
, skip_validation
=False, skip_charm_build
=False):
193 **Creates a .tar.gz file given a package_folder**
196 - package_folder: is the name of the folder to be packaged
197 - skip_validation: is the flag to validate or not the descriptors on the folder before build
199 :returns: message result for the build process
201 self
._logger
.debug("")
202 package_folder
= package_folder
.rstrip('/')
203 if not os
.path
.exists("{}".format(package_folder
)):
204 return "Fail, package is not in the specified path"
205 if not skip_validation
:
206 print('Validating package {}'.format(package_folder
))
207 results
= self
.validate(package_folder
, recursive
=False)
209 for result
in results
:
210 if result
["valid"] != "OK":
211 raise ClientException("There was an error validating the file {} with error: {}"
212 .format(result
["path"], result
["error"]))
213 print('Validation OK')
215 raise ClientException("No descriptor file found in: {}".format(package_folder
))
216 charm_list
= self
.build_all_charms(package_folder
, skip_charm_build
)
217 return self
.build_tarfile(package_folder
, charm_list
)
219 def calculate_checksum(self
, package_folder
):
221 **Function to calculate the checksum given a folder**
224 - package_folder: is the folder where we have the files to calculate the checksum
227 self
._logger
.debug("")
228 files
= [f
for f
in glob
.glob(package_folder
+ "/**/*.*", recursive
=True) if os
.path
.isfile(f
)]
229 with
open("{}/checksums.txt".format(package_folder
), "w+") as checksum
:
230 for file_item
in files
:
231 if "checksums.txt" in file_item
:
233 # from https://www.quickprogrammingtips.com/python/how-to-calculate-md5-hash-of-a-file-in-python.html
234 md5_hash
= hashlib
.md5()
235 with
open(file_item
, "rb") as f
:
236 # Read and update hash in chunks of 4K
237 for byte_block
in iter(lambda: f
.read(4096), b
""):
238 md5_hash
.update(byte_block
)
239 checksum
.write("{}\t{}\n".format(md5_hash
.hexdigest(), file_item
))
241 def create_folders(self
, folders
, package_type
):
243 **Create folder given a list of folders**
246 - folders: [List] list of folders paths to be created
247 - package_type: is the type of package to be created
250 self
._logger
.debug("")
251 for folder
in folders
:
253 # print("Folder {} == package_type {}".format(folder[1], package_type))
254 if folder
[1] == package_type
:
255 print("Creating folder:\t{}".format(folder
[0]))
256 os
.makedirs(folder
[0])
257 except FileExistsError
:
260 def save_file(self
, file_name
, file_body
):
262 **Create a file given a name and the content**
265 - file_name: is the name of the file with the relative route
266 - file_body: is the content of the file
269 self
._logger
.debug("")
270 print("Creating file: \t{}".format(file_name
))
272 with
open(file_name
, "w+") as f
:
274 except Exception as e
:
275 raise ClientException(e
)
277 def generate_readme(self
):
279 **Creates the README content**
281 :returns: readme content
283 self
._logger
.debug("")
284 return """# Descriptor created by OSM descriptor package generated\n\n**Created on {} **""".format(
285 time
.strftime("%m/%d/%Y, %H:%M:%S", time
.localtime()))
287 def generate_cloud_init(self
):
289 **Creates the cloud-init content**
291 :returns: cloud-init content
293 self
._logger
.debug("")
294 return "---\n#cloud-config"
296 def create_files(self
, files
, file_content
, package_type
):
298 **Creates the files given the file list and type**
301 - files: is the list of files structure
302 - file_content: is the content of the descriptor rendered by the template
303 - package_type: is the type of package to filter the creation structure
307 self
._logger
.debug("")
308 for file_item
, file_package
, file_type
in files
:
309 if package_type
== file_package
:
310 if file_type
== "descriptor":
311 self
.save_file(file_item
, file_content
)
312 elif file_type
== "readme":
313 self
.save_file(file_item
, self
.generate_readme())
314 elif file_type
== "cloud_init":
315 self
.save_file(file_item
, self
.generate_cloud_init())
317 def check_files_folders(self
, path_list
, override
):
319 **Find files and folders missing given a directory structure {"folders": [], "files": []}**
322 - path_list: is the list of files and folders to be created
323 - override: is the flag used to indicate the creation of the list even if the file exist to override it
325 :return: Missing paths Dict
327 self
._logger
.debug("")
331 for folder
in path_list
.get("folders"):
332 if not os
.path
.exists(folder
[0]):
333 folders
.append(folder
)
334 missing_paths
["folders"] = folders
336 for file_item
in path_list
.get("files"):
337 if not os
.path
.exists(file_item
[0]) or override
is True:
338 files
.append(file_item
)
339 missing_paths
["files"] = files
343 def build_all_charms(self
, package_folder
, skip_charm_build
):
345 **Read the descriptor file, check that the charms referenced are in the folder and compiles them**
348 - packet_folder: is the location of the package
349 :return: Files and Folders not found. In case of override, it will return all file list
351 self
._logger
.debug("")
353 descriptor_file
= False
354 descriptors_paths
= [f
for f
in glob
.glob(package_folder
+ "/*.yaml")]
355 for file in descriptors_paths
:
356 if file.endswith('nfd.yaml'):
357 descriptor_file
= True
358 listCharms
= self
.charms_search(file, 'vnf')
359 if file.endswith('nsd.yaml'):
360 descriptor_file
= True
361 listCharms
= self
.charms_search(file, 'ns')
362 print("List of charms in the descriptor: {}".format(listCharms
))
363 if not descriptor_file
:
364 raise ClientException('Descriptor filename is not correct in: {}. It should end with "nfd.yaml" or "nsd.yaml"'.format(package_folder
))
365 if listCharms
and not skip_charm_build
:
366 for charmName
in listCharms
:
367 if os
.path
.isdir('{}/charms/layers/{}'.format(package_folder
, charmName
)):
368 print('Building charm {}/charms/layers/{}'.format(package_folder
, charmName
))
369 self
.charm_build(package_folder
, charmName
)
370 print('Charm built: {}'.format(charmName
))
372 if not os
.path
.isdir('{}/charms/{}'.format(package_folder
, charmName
)):
373 raise ClientException('The charm: {} referenced in the descriptor file '
374 'is not present either in {}/charms or in {}/charms/layers'.
375 format(charmName
, package_folder
, package_folder
))
376 self
._logger
.debug("Return list of charms: {}".format(listCharms
))
379 def discover_folder_structure(self
, base_directory
, name
, override
):
381 **Discover files and folders structure for OSM descriptors given a base_directory and name**
384 - base_directory: is the location of the package to be created
385 - name: is the name of the package
386 - override: is the flag used to indicate the creation of the list even if the file exist to override it
387 :return: Files and Folders not found. In case of override, it will return all file list
389 self
._logger
.debug("")
390 prefix
= "{}/{}".format(base_directory
, name
)
391 files_folders
= {"folders": [("{}_ns".format(prefix
), "ns"),
392 ("{}_ns/icons".format(prefix
), "ns"),
393 ("{}_ns/charms".format(prefix
), "ns"),
394 ("{}_vnf".format(name
), "vnf"),
395 ("{}_vnf/charms".format(prefix
), "vnf"),
396 ("{}_vnf/cloud_init".format(prefix
), "vnf"),
397 ("{}_vnf/images".format(prefix
), "vnf"),
398 ("{}_vnf/icons".format(prefix
), "vnf"),
399 ("{}_vnf/scripts".format(prefix
), "vnf"),
400 ("{}_nst".format(prefix
), "nst"),
401 ("{}_nst/icons".format(prefix
), "nst")
403 "files": [("{}_ns/{}_nsd.yaml".format(prefix
, name
), "ns", "descriptor"),
404 ("{}_ns/README.md".format(prefix
), "ns", "readme"),
405 ("{}_vnf/{}_vnfd.yaml".format(prefix
, name
), "vnf", "descriptor"),
406 ("{}_vnf/cloud_init/cloud-config.txt".format(prefix
), "vnf", "cloud_init"),
407 ("{}_vnf/README.md".format(prefix
), "vnf", "readme"),
408 ("{}_nst/{}_nst.yaml".format(prefix
, name
), "nst", "descriptor"),
409 ("{}_nst/README.md".format(prefix
), "nst", "readme")
412 missing_files_folders
= self
.check_files_folders(files_folders
, override
)
413 # print("Missing files and folders: {}".format(missing_files_folders))
414 return missing_files_folders
416 def charm_build(self
, charms_folder
, build_name
):
418 Build the charms inside the package.
419 params: package_folder is the name of the folder where is the charms to compile.
420 build_name is the name of the layer or interface
422 self
._logger
.debug("")
423 os
.environ
['JUJU_REPOSITORY'] = "{}/charms".format(charms_folder
)
424 os
.environ
['CHARM_LAYERS_DIR'] = "{}/layers".format(os
.environ
['JUJU_REPOSITORY'])
425 os
.environ
['CHARM_INTERFACES_DIR'] = "{}/interfaces".format(os
.environ
['JUJU_REPOSITORY'])
426 os
.environ
['CHARM_BUILD_DIR'] = "{}/charms/builds".format(charms_folder
)
427 if not os
.path
.exists(os
.environ
['CHARM_BUILD_DIR']):
428 os
.makedirs(os
.environ
['CHARM_BUILD_DIR'])
429 src_folder
= '{}/{}'.format(os
.environ
['CHARM_LAYERS_DIR'], build_name
)
430 result
= subprocess
.run(["charm", "build", "{}".format(src_folder
)])
431 if result
.returncode
== 1:
432 raise ClientException("failed to build the charm: {}".format(src_folder
))
433 self
._logger
.verbose("charm {} built".format(src_folder
))
435 def build_tarfile(self
, package_folder
, charm_list
=None):
437 Creates a .tar.gz file given a package_folder
438 params: package_folder is the name of the folder to be packaged
439 returns: .tar.gz name
441 self
._logger
.debug("")
444 directory_name
, package_name
= self
.create_temp_dir(package_folder
, charm_list
)
446 os
.chdir(directory_name
)
447 self
.calculate_checksum(package_name
)
448 with tarfile
.open("{}.tar.gz".format(package_name
), mode
='w:gz') as archive
:
449 print("Adding File: {}".format(package_name
))
450 archive
.add('{}'.format(package_name
), recursive
=True)
451 # return "Created {}.tar.gz".format(package_folder)
452 # self.build("{}".format(os.path.basename(package_folder)))
455 created_package
= "{}/{}.tar.gz".format(os
.path
.dirname(package_folder
) or '.', package_name
)
456 os
.rename("{}/{}.tar.gz".format(directory_name
, package_name
),
458 os
.rename("{}/{}/checksums.txt".format(directory_name
, package_name
),
459 "{}/checksums.txt".format(package_folder
))
460 print("Package created: {}".format(created_package
))
461 return created_package
462 except Exception as exc
:
463 raise ClientException('failure during build of targz file (create temp dir, calculate checksum, '
464 'tar.gz file): {}'.format(exc
))
468 shutil
.rmtree(os
.path
.join(package_folder
, "tmp"))
470 def create_temp_dir(self
, package_folder
, charm_list
=None):
472 Method to create a temporary folder where we can move the files in package_folder
474 self
._logger
.debug("")
475 ignore_patterns
= ('.gitignore')
476 ignore
= shutil
.ignore_patterns(ignore_patterns
)
477 directory_name
= os
.path
.abspath(package_folder
)
478 package_name
= os
.path
.basename(directory_name
)
479 directory_name
+= "/tmp"
480 os
.makedirs("{}/{}".format(directory_name
, package_name
), exist_ok
=True)
481 self
._logger
.debug("Makedirs DONE: {}/{}".format(directory_name
, package_name
))
482 for item
in os
.listdir(package_folder
):
483 self
._logger
.debug("Item: {}".format(item
))
485 s
= os
.path
.join(package_folder
, item
)
486 d
= os
.path
.join(os
.path
.join(directory_name
, package_name
), item
)
489 os
.makedirs(d
, exist_ok
=True)
490 s_builds
= os
.path
.join(s
, "builds")
491 for charm
in charm_list
:
492 self
._logger
.debug("Copying charm {}".format(charm
))
493 if charm
in os
.listdir(s
):
494 s_charm
= os
.path
.join(s
, charm
)
495 elif charm
in os
.listdir(s_builds
):
496 s_charm
= os
.path
.join(s_builds
, charm
)
498 raise ClientException('The charm {} referenced in the descriptor file '
499 'could not be found in {}/charms or in {}/charms/builds'.
500 format(charm
, package_folder
, package_folder
))
501 d_temp
= os
.path
.join(d
, charm
)
502 self
._logger
.debug("Copying tree: {} -> {}".format(s_charm
, d_temp
))
503 shutil
.copytree(s_charm
, d_temp
, symlinks
=True, ignore
=ignore
)
504 self
._logger
.debug("DONE")
506 self
._logger
.debug("Copying tree: {} -> {}".format(s
, d
))
507 shutil
.copytree(s
, d
, symlinks
=True, ignore
=ignore
)
508 self
._logger
.debug("DONE")
510 if item
in ignore_patterns
:
512 self
._logger
.debug("Copying file: {} -> {}".format(s
, d
))
514 self
._logger
.debug("DONE")
515 return directory_name
, package_name
517 def charms_search(self
, descriptor_file
, desc_type
):
518 self
._logger
.debug("descriptor_file: {}, desc_type: {}".format(descriptor_file
,
520 with
open("{}".format(descriptor_file
)) as yaml_desc
:
521 descriptor_dict
= yaml
.safe_load(yaml_desc
)
522 #self._logger.debug("\n"+yaml.safe_dump(descriptor_dict, indent=4, default_flow_style=False))
524 if ( (desc_type
=="vnf" and ("vnfd:vnfd-catalog" in descriptor_dict
or "vnfd-catalog" in descriptor_dict
)) or
525 (desc_type
=="ns" and ( "nsd:nsd-catalog" in descriptor_dict
or "nsd-catalog" in descriptor_dict
)) ):
526 charms_list
= self
._charms
_search
_on
_osm
_im
_dict
(descriptor_dict
, desc_type
)
528 if desc_type
== "ns":
529 get_charm_list
= self
._charms
_search
_on
_nsd
_sol
006_dict
530 elif desc_type
== "vnf":
531 get_charm_list
= self
._charms
_search
_on
_vnfd
_sol
006_dict
533 raise Exception("Bad descriptor type")
534 charms_list
= get_charm_list(descriptor_dict
)
537 def _charms_search_on_osm_im_dict(self
, osm_im_dict
, desc_type
):
538 self
._logger
.debug("")
540 for k1
, v1
in osm_im_dict
.items():
541 for k2
, v2
in v1
.items():
543 if '{}-configuration'.format(desc_type
) in entry
:
544 vnf_config
= entry
['{}-configuration'.format(desc_type
)]
545 for k3
, v3
in vnf_config
.items():
547 charms_list
.append((v3
['charm']))
551 if 'vdu-configuration' in vdu
:
552 for k4
, v4
in vdu
['vdu-configuration'].items():
554 charms_list
.append((v4
['charm']))
557 def _charms_search_on_vnfd_sol006_dict(self
, sol006_dict
):
558 self
._logger
.debug("")
560 for k1
, v1
in sol006_dict
.items():
561 for k2
, v2
in v1
.items():
564 lcm_ops
= df
.get("lcm-operations-configuration", {})
565 ops_config
= lcm_ops
.get("operate-vnf-op-config", {})
566 for day_12_config
in ops_config
.get("day1-2", []):
567 self
._logger
.debug("Execution environment found")
568 for ee
in day_12_config
.get("execution-environment-list", []):
570 charms_list
.append((ee
["juju"]['charm']))
573 def _charms_search_on_nsd_sol006_dict(self
, sol006_dict
):
574 self
._logger
.debug("")
576 nsd_list
= sol006_dict
.get("nsd", {}).get("nsd", [])
578 charm
= nsd
.get("ns-configuration", {}).get("juju", {}).get("charm")
580 charms_list
.append(charm
)