72d7d3dcfce5f942351bb0dc781fecfa54ffa8b8
6 # Licensed under the Apache License, Version 2.0 (the "License"); you may
7 # not use this file except in compliance with the License. You may obtain
8 # a copy of the License at
10 # http://www.apache.org/licenses/LICENSE-2.0
12 # Unless required by applicable law or agreed to in writing, software
13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15 # License for the specific language governing permissions and limitations
18 from osmclient
.common
.exceptions
import ClientException
24 from osm_im
.validation
import Validation
as validation_im
25 from jinja2
import Environment
, PackageLoader
31 class PackageTool(object):
32 def __init__(self
, client
=None):
34 self
._logger
= logging
.getLogger('osmclient')
36 def create(self
, package_type
, base_directory
, package_name
, override
, image
, vdus
, vcpu
, memory
, storage
,
37 interfaces
, vendor
, detailed
, netslice_subnets
, netslice_vlds
):
39 **Create a package descriptor**
42 - package_type: [vnf, ns, nst]
43 - base directory: path of destination folder
44 - package_name: is the name of the package to be created
45 - image: specify the image of the vdu
46 - vcpu: number of virtual cpus of the vdu
47 - memory: amount of memory in MB pf the vdu
48 - storage: amount of storage in GB of the vdu
49 - interfaces: number of interfaces besides management interface
50 - vendor: vendor name of the vnf/ns
51 - detailed: include all possible values for NSD, VNFD, NST
52 - netslice_subnets: number of netslice_subnets for the NST
53 - netslice_vlds: number of virtual link descriptors for the NST
57 self
._logger
.debug("")
58 # print("location: {}".format(osmclient.__path__))
59 file_loader
= PackageLoader("osmclient")
60 env
= Environment(loader
=file_loader
)
61 if package_type
== 'ns':
62 template
= env
.get_template('nsd.yaml.j2')
63 content
= {"name": package_name
, "vendor": vendor
, "vdus": vdus
, "clean": False, "interfaces": interfaces
,
65 elif package_type
== 'vnf':
66 template
= env
.get_template('vnfd.yaml.j2')
67 content
= {"name": package_name
, "vendor": vendor
, "vdus": vdus
, "clean": False, "interfaces": interfaces
,
68 "image": image
, "vcpu": vcpu
, "memory": memory
, "storage": storage
, "detailed": detailed
}
69 elif package_type
== 'nst':
70 template
= env
.get_template('nst.yaml.j2')
71 content
= {"name": package_name
, "vendor": vendor
, "interfaces": interfaces
,
72 "netslice_subnets": netslice_subnets
, "netslice_vlds": netslice_vlds
, "detailed": detailed
}
74 raise ClientException("Wrong descriptor type {}. Options: ns, vnf, nst".format(package_type
))
76 # print("To be rendered: {}".format(content))
77 output
= template
.render(content
)
80 structure
= self
.discover_folder_structure(base_directory
, package_name
, override
)
81 if structure
.get("folders"):
82 self
.create_folders(structure
["folders"], package_type
)
83 if structure
.get("files"):
84 self
.create_files(structure
["files"], output
, package_type
)
87 def validate(self
, base_directory
, recursive
=True):
89 **Validate OSM Descriptors given a path**
92 - base_directory is the root path for all descriptors
94 :return: List of dict of validated descriptors. keys: type, path, valid, error
96 self
._logger
.debug("")
99 descriptors_paths
= [f
for f
in glob
.glob(base_directory
+ "/**/*.yaml", recursive
=recursive
)]
101 descriptors_paths
= [f
for f
in glob
.glob(base_directory
+ "/*.yaml", recursive
=recursive
)]
102 print("Base directory: {}".format(base_directory
))
103 print("{} Descriptors found to validate".format(len(descriptors_paths
)))
104 for desc_path
in descriptors_paths
:
105 with
open(desc_path
) as descriptor_file
:
106 descriptor_data
= descriptor_file
.read()
109 desc_type
, descriptor_data
= validation_im
.yaml_validation(self
, descriptor_data
)
110 validation_im
.pyangbind_validation(self
, desc_type
, descriptor_data
)
111 table
.append({"type": desc_type
, "path": desc_path
, "valid": "OK", "error": "-"})
112 except Exception as e
:
113 table
.append({"type": desc_type
, "path": desc_path
, "valid": "ERROR", "error": str(e
)})
116 def build(self
, package_folder
, skip_validation
=False, skip_charm_build
=False):
118 **Creates a .tar.gz file given a package_folder**
121 - package_folder: is the name of the folder to be packaged
122 - skip_validation: is the flag to validate or not the descriptors on the folder before build
124 :returns: message result for the build process
126 self
._logger
.debug("")
127 package_folder
= package_folder
.rstrip('/')
128 if not os
.path
.exists("{}".format(package_folder
)):
129 return "Fail, package is not in the specified route"
130 if not skip_validation
:
131 results
= self
.validate(package_folder
, recursive
=False)
133 for result
in results
:
134 if result
["valid"] != "OK":
135 raise ClientException("There was an error validating the file: {} with error: {}"
136 .format(result
["path"], result
["error"]))
138 raise ClientException("No descriptor file found in: {}".format(package_folder
))
139 charm_list
= self
.build_all_charms(package_folder
, skip_charm_build
)
140 return self
.build_tarfile(package_folder
, charm_list
)
142 def calculate_checksum(self
, package_folder
):
144 **Function to calculate the checksum given a folder**
147 - package_folder: is the folder where we have the files to calculate the checksum
150 self
._logger
.debug("")
151 files
= [f
for f
in glob
.glob(package_folder
+ "/**/*.*", recursive
=True)]
152 checksum
= open("{}/checksums.txt".format(package_folder
), "w+")
153 for file_item
in files
:
154 if "checksums.txt" in file_item
:
156 # from https://www.quickprogrammingtips.com/python/how-to-calculate-md5-hash-of-a-file-in-python.html
157 md5_hash
= hashlib
.md5()
158 with
open(file_item
, "rb") as f
:
159 # Read and update hash in chunks of 4K
160 for byte_block
in iter(lambda: f
.read(4096), b
""):
161 md5_hash
.update(byte_block
)
162 checksum
.write("{}\t{}\n".format(md5_hash
.hexdigest(), file_item
))
165 def create_folders(self
, folders
, package_type
):
167 **Create folder given a list of folders**
170 - folders: [List] list of folders paths to be created
171 - package_type: is the type of package to be created
175 for folder
in folders
:
177 # print("Folder {} == package_type {}".format(folder[1], package_type))
178 if folder
[1] == package_type
:
179 print("Creating folder:\t{}".format(folder
[0]))
180 os
.makedirs(folder
[0])
181 except FileExistsError
:
184 def save_file(self
, file_name
, file_body
):
186 **Create a file given a name and the content**
189 - file_name: is the name of the file with the relative route
190 - file_body: is the content of the file
193 print("Creating file: \t{}".format(file_name
))
195 with
open(file_name
, "w+") as f
:
197 except Exception as e
:
198 raise ClientException(e
)
200 def generate_readme(self
):
202 **Creates the README content**
204 :returns: readme content
206 return """# Descriptor created by OSM descriptor package generated\n\n**Created on {} **""".format(
207 time
.strftime("%m/%d/%Y, %H:%M:%S", time
.localtime()))
209 def generate_cloud_init(self
):
211 **Creates the cloud-init content**
213 :returns: cloud-init content
215 return "---\n#cloud-config"
217 def create_files(self
, files
, file_content
, package_type
):
219 **Creates the files given the file list and type**
222 - files: is the list of files structure
223 - file_content: is the content of the descriptor rendered by the template
224 - package_type: is the type of package to filter the creation structure
228 self
._logger
.debug("")
229 for file_item
, file_package
, file_type
in files
:
230 if package_type
== file_package
:
231 if file_type
== "descriptor":
232 self
.save_file(file_item
, file_content
)
233 elif file_type
== "readme":
234 self
.save_file(file_item
, self
.generate_readme())
235 elif file_type
== "cloud_init":
236 self
.save_file(file_item
, self
.generate_cloud_init())
238 def check_files_folders(self
, path_list
, override
):
240 **Find files and folders missing given a directory structure {"folders": [], "files": []}**
243 - path_list: is the list of files and folders to be created
244 - override: is the flag used to indicate the creation of the list even if the file exist to override it
246 :return: Missing paths Dict
248 self
._logger
.debug("")
252 for folder
in path_list
.get("folders"):
253 if not os
.path
.exists(folder
[0]):
254 folders
.append(folder
)
255 missing_paths
["folders"] = folders
257 for file_item
in path_list
.get("files"):
258 if not os
.path
.exists(file_item
[0]) or override
is True:
259 files
.append(file_item
)
260 missing_paths
["files"] = files
264 def build_all_charms(self
, package_folder
, skip_charm_build
):
266 **Read the descriptor file, check that the charms referenced are in the folder and compiles them**
269 - packet_folder: is the location of the package
270 :return: Files and Folders not found. In case of override, it will return all file list
273 self
._logger
.debug("")
274 descriptor_file
= False
275 descriptors_paths
= [f
for f
in glob
.glob(package_folder
+ "/*.yaml")]
276 for file in descriptors_paths
:
277 if 'nfd.yaml' in file:
278 descriptor_file
= True
279 listCharms
= self
.charms_search(file, 'vnf')
280 if 'nsd.yaml' in file:
281 descriptor_file
= True
282 listCharms
= self
.charms_search(file, 'ns')
283 if not descriptor_file
:
284 raise ClientException ('descriptor name is not correct in: {}'.format(package_folder
))
285 if listCharms
and not skip_charm_build
:
286 for charmName
in listCharms
:
287 if os
.path
.isdir('{}/charms/layers/{}'.format(package_folder
,charmName
)):
288 self
.charm_build(package_folder
, charmName
)
290 if not os
.path
.isdir('{}/charms/{}'.format(package_folder
,charmName
)):
291 raise ClientException ('The charm: {} referenced in the descriptor file '
292 'is not present either in {}/charms or in {}/charms/layers'.
293 format(charmName
, package_folder
,package_folder
))
296 def discover_folder_structure(self
, base_directory
, name
, override
):
298 **Discover files and folders structure for OSM descriptors given a base_directory and name**
301 - base_directory: is the location of the package to be created
302 - name: is the name of the package
303 - override: is the flag used to indicate the creation of the list even if the file exist to override it
304 :return: Files and Folders not found. In case of override, it will return all file list
306 self
._logger
.debug("")
307 prefix
= "{}/{}".format(base_directory
, name
)
308 files_folders
= {"folders": [("{}_ns".format(prefix
), "ns"),
309 ("{}_ns/icons".format(prefix
), "ns"),
310 ("{}_ns/charms".format(prefix
), "ns"),
311 ("{}_vnf".format(name
), "vnf"),
312 ("{}_vnf/charms".format(prefix
), "vnf"),
313 ("{}_vnf/cloud_init".format(prefix
), "vnf"),
314 ("{}_vnf/images".format(prefix
), "vnf"),
315 ("{}_vnf/icons".format(prefix
), "vnf"),
316 ("{}_vnf/scripts".format(prefix
), "vnf"),
317 ("{}_nst".format(prefix
), "nst"),
318 ("{}_nst/icons".format(prefix
), "nst")
320 "files": [("{}_ns/{}_nsd.yaml".format(prefix
, name
), "ns", "descriptor"),
321 ("{}_ns/README.md".format(prefix
), "ns", "readme"),
322 ("{}_vnf/{}_vnfd.yaml".format(prefix
, name
), "vnf", "descriptor"),
323 ("{}_vnf/cloud_init/cloud-config.txt".format(prefix
), "vnf", "cloud_init"),
324 ("{}_vnf/README.md".format(prefix
), "vnf", "readme"),
325 ("{}_nst/{}_nst.yaml".format(prefix
, name
), "nst", "descriptor"),
326 ("{}_nst/README.md".format(prefix
), "nst", "readme")
329 missing_files_folders
= self
.check_files_folders(files_folders
, override
)
330 # print("Missing files and folders: {}".format(missing_files_folders))
331 return missing_files_folders
333 def charm_build(self
, charms_folder
, build_name
):
335 Build the charms inside the package.
336 params: package_folder is the name of the folder where is the charms to compile.
337 build_name is the name of the layer or interface
339 self
._logger
.debug("")
340 os
.environ
['JUJU_REPOSITORY'] = "{}/charms".format(charms_folder
)
341 os
.environ
['CHARM_LAYERS_DIR'] = "{}/layers".format(os
.environ
['JUJU_REPOSITORY'])
342 os
.environ
['CHARM_INTERFACES_DIR'] = "{}/interfaces".format(os
.environ
['JUJU_REPOSITORY'])
343 os
.environ
['CHARM_BUILD_DIR'] = "{}/charms/builds".format(charms_folder
)
344 src_folder
= '{}/{}'.format(os
.environ
['CHARM_LAYERS_DIR'], build_name
)
345 result
= subprocess
.run(["charm", "build", "{}".format(src_folder
)])
346 if result
.returncode
== 1:
347 raise ClientException("failed to build the charm: {}".format(src_folder
))
348 self
._logger
.verbose("charm: {} compiled".format(src_folder
))
350 def build_tarfile(self
, package_folder
, charm_list
=None):
352 Creates a .tar.gz file given a package_folder
353 params: package_folder is the name of the folder to be packaged
354 returns: .tar.gz name
356 self
._logger
.debug("")
357 ignore_patterns
= "'*layers*', '*interfaces*'"
359 directory_name
= self
.create_temp_dir(package_folder
, ignore_patterns
, charm_list
)
361 os
.chdir(directory_name
)
362 self
.calculate_checksum(package_folder
)
363 with tarfile
.open("{}.tar.gz".format(package_folder
), mode
='w:gz') as archive
:
364 print("Adding File: {}".format(package_folder
))
365 archive
.add('{}'.format(package_folder
), recursive
=True)
366 #return "Created {}.tar.gz".format(package_folder)
367 #self.build("{}".format(os.path.basename(package_folder)))
370 shutil
.rmtree(os
.path
.join(package_folder
, "tmp"))
371 raise ClientException('failure to manipulate the result of the compilation')
372 os
.rename("{}/{}.tar.gz".format(directory_name
, os
.path
.basename(package_folder
)),
373 "{}.tar.gz".format(os
.path
.basename(package_folder
)))
374 os
.rename("{}/{}/checksums.txt".format(directory_name
, os
.path
.basename(package_folder
)),
375 "{}/checksums.txt".format(package_folder
))
376 shutil
.rmtree(os
.path
.join(package_folder
, "tmp"))
377 self
._logger
.verbose("package created: {}.tar.gz".format(os
.path
.basename(package_folder
)))
378 return "{}.tar.gz".format(package_folder
)
380 def create_temp_dir(self
, package_folder
, ignore_patterns
=None, charm_list
=None):
382 Method to create a temporary folder where we can move the files in package_folder, which do not
383 meet the pattern defined in ignore_patterns
385 self
._logger
.debug("")
386 ignore
= shutil
.ignore_patterns(ignore_patterns
)
387 os
.makedirs("{}/tmp".format(package_folder
), exist_ok
=True)
388 directory_name
= os
.path
.abspath("{}/tmp".format(package_folder
))
389 os
.makedirs("{}/{}".format(directory_name
, os
.path
.basename(package_folder
),exist_ok
=True))
390 for item
in os
.listdir(package_folder
):
392 s
= os
.path
.join(package_folder
, item
)
393 d
= os
.path
.join(os
.path
.join(directory_name
, os
.path
.basename(package_folder
)), item
)
396 s
= os
.path
.join(s
, "builds")
397 if not os
.path
.exists(s
):
399 for i
in os
.listdir(s
):
401 s_charm
= os
.path
.join(s
, i
)
402 # d_charm = os.path.join(package_folder, item, i)
403 d_temp
= os
.path
.join(d
, i
)
404 # if os.path.exists(d_charm):
405 # shutil.rmtree(d_charm)
406 shutil
.copytree(s_charm
, d_temp
, symlinks
= True, ignore
= ignore
)
407 # shutil.copytree(s_charm, d_charm, symlinks = True, ignore = ignore)
409 shutil
.copytree(s
, d
, symlinks
= True, ignore
= ignore
)
412 return directory_name
414 def charms_search(self
, descriptor_file
, desc_type
):
415 self
._logger
.debug("")
418 with
open("{}".format(descriptor_file
)) as yaml_desc
:
419 dict = yaml
.safe_load(yaml_desc
)
420 for k1
, v1
in dict.items():
421 for k2
, v2
in v1
.items():
423 if '{}-configuration'.format(desc_type
) in entry
:
424 name
= entry
['{}-configuration'.format(desc_type
)]
425 for k3
, v3
in name
.items():
427 list.append((v3
['charm']))
431 if 'vdu-configuration' in vdu
:
432 for k4
, v4
in vdu
['vdu-configuration'].items():
434 list.append((v4
['charm']))