| 1 |
|
# /bin/env python3 |
| 2 |
|
# Copyright 2019 ATOS |
| 3 |
|
# |
| 4 |
|
# All Rights Reserved. |
| 5 |
|
# |
| 6 |
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
| 7 |
|
# not use this file except in compliance with the License. You may obtain |
| 8 |
|
# a copy of the License at |
| 9 |
|
# |
| 10 |
|
# http://www.apache.org/licenses/LICENSE-2.0 |
| 11 |
|
# |
| 12 |
|
# Unless required by applicable law or agreed to in writing, software |
| 13 |
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 14 |
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| 15 |
|
# License for the specific language governing permissions and limitations |
| 16 |
|
# under the License. |
| 17 |
|
|
| 18 |
1 |
import glob |
| 19 |
1 |
import hashlib |
| 20 |
1 |
import logging |
| 21 |
1 |
import os |
| 22 |
1 |
import shutil |
| 23 |
1 |
import subprocess |
| 24 |
1 |
import tarfile |
| 25 |
1 |
import time |
| 26 |
|
|
| 27 |
1 |
from jinja2 import Environment, PackageLoader |
| 28 |
1 |
from osm_im.validation import Validation as validation_im |
| 29 |
1 |
from osm_im.validation import ValidationException |
| 30 |
1 |
from osm_im import im_translation |
| 31 |
1 |
from osmclient.common.exceptions import ClientException |
| 32 |
1 |
import yaml |
| 33 |
|
|
| 34 |
|
|
| 35 |
1 |
class PackageTool(object): |
| 36 |
1 |
def __init__(self, client=None): |
| 37 |
1 |
self._client = client |
| 38 |
1 |
self._logger = logging.getLogger('osmclient') |
| 39 |
1 |
self._validator = validation_im() |
| 40 |
|
|
| 41 |
1 |
def create(self, package_type, base_directory, package_name, override, image, vdus, vcpu, memory, storage, |
| 42 |
|
interfaces, vendor, detailed, netslice_subnets, netslice_vlds, old): |
| 43 |
|
""" |
| 44 |
|
**Create a package descriptor** |
| 45 |
|
|
| 46 |
|
:params: |
| 47 |
|
- package_type: [vnf, ns, nst] |
| 48 |
|
- base directory: path of destination folder |
| 49 |
|
- package_name: is the name of the package to be created |
| 50 |
|
- image: specify the image of the vdu |
| 51 |
|
- vcpu: number of virtual cpus of the vdu |
| 52 |
|
- memory: amount of memory in MB pf the vdu |
| 53 |
|
- storage: amount of storage in GB of the vdu |
| 54 |
|
- interfaces: number of interfaces besides management interface |
| 55 |
|
- vendor: vendor name of the vnf/ns |
| 56 |
|
- detailed: include all possible values for NSD, VNFD, NST |
| 57 |
|
- netslice_subnets: number of netslice_subnets for the NST |
| 58 |
|
- netslice_vlds: number of virtual link descriptors for the NST |
| 59 |
|
- old: flag to create a package using the IM of OSM<9 |
| 60 |
|
|
| 61 |
|
:return: status |
| 62 |
|
""" |
| 63 |
0 |
self._logger.debug("") |
| 64 |
|
# print("location: {}".format(osmclient.__path__)) |
| 65 |
0 |
file_loader = PackageLoader("osmclient") |
| 66 |
0 |
env = Environment(loader=file_loader) |
| 67 |
0 |
if package_type == 'ns': |
| 68 |
0 |
template = env.get_template("nsd.yaml.j2" if not old else "nsd_old.yaml.j2") |
| 69 |
0 |
content = {"name": package_name, "vendor": vendor, "vdus": vdus, "clean": False, "interfaces": interfaces, |
| 70 |
|
"detailed": detailed} |
| 71 |
0 |
elif package_type == 'vnf': |
| 72 |
0 |
template = env.get_template("vnfd.yaml.j2" if not old else "vnfd_old.yaml.j2") |
| 73 |
0 |
content = {"name": package_name, "vendor": vendor, "vdus": vdus, "clean": False, "interfaces": interfaces, |
| 74 |
|
"image": image, "vcpu": vcpu, "memory": memory, "storage": storage, "detailed": detailed} |
| 75 |
0 |
elif package_type == 'nst': |
| 76 |
0 |
template = env.get_template('nst.yaml.j2') |
| 77 |
0 |
content = {"name": package_name, "vendor": vendor, "interfaces": interfaces, |
| 78 |
|
"netslice_subnets": netslice_subnets, "netslice_vlds": netslice_vlds, "detailed": detailed} |
| 79 |
|
else: |
| 80 |
0 |
raise ClientException("Wrong descriptor type {}. Options: ns, vnf, nst".format(package_type)) |
| 81 |
|
|
| 82 |
|
# print("To be rendered: {}".format(content)) |
| 83 |
0 |
output = template.render(content) |
| 84 |
|
# print(output) |
| 85 |
|
|
| 86 |
0 |
structure = self.discover_folder_structure(base_directory, package_name, override) |
| 87 |
0 |
if structure.get("folders"): |
| 88 |
0 |
self.create_folders(structure["folders"], package_type) |
| 89 |
0 |
if structure.get("files"): |
| 90 |
0 |
self.create_files(structure["files"], output, package_type) |
| 91 |
0 |
return "Created" |
| 92 |
|
|
| 93 |
1 |
def validate(self, base_directory, recursive=True, old_format=False): |
| 94 |
|
""" |
| 95 |
|
**Validate OSM Descriptors given a path** |
| 96 |
|
|
| 97 |
|
:params: |
| 98 |
|
- base_directory is the root path for all descriptors |
| 99 |
|
|
| 100 |
|
:return: List of dict of validated descriptors. keys: type, path, valid, error |
| 101 |
|
""" |
| 102 |
0 |
self._logger.debug("") |
| 103 |
0 |
table = [] |
| 104 |
0 |
if recursive: |
| 105 |
0 |
descriptors_paths = [f for f in glob.glob(base_directory + "/**/*.yaml", recursive=recursive)] |
| 106 |
|
else: |
| 107 |
0 |
descriptors_paths = [f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)] |
| 108 |
0 |
print("Base directory: {}".format(base_directory)) |
| 109 |
0 |
print("{} Descriptors found to validate".format(len(descriptors_paths))) |
| 110 |
0 |
for desc_path in descriptors_paths: |
| 111 |
0 |
with open(desc_path) as descriptor_file: |
| 112 |
0 |
descriptor_data = descriptor_file.read() |
| 113 |
0 |
desc_type = "-" |
| 114 |
0 |
try: |
| 115 |
0 |
desc_type, descriptor_data = validation_im.yaml_validation(self, descriptor_data) |
| 116 |
0 |
if not old_format: |
| 117 |
0 |
if ( desc_type=="vnfd" or desc_type=="nsd" ): |
| 118 |
0 |
print("OSM descriptor '{}' written in an unsupported format. Please update to ETSI SOL006 format".format(desc_path)) |
| 119 |
0 |
print("Package validation skipped. It can still be done with 'osm package-validate --old'") |
| 120 |
0 |
print("Package build can still be done with 'osm package-build --skip-validation'") |
| 121 |
0 |
raise Exception("Not SOL006 format") |
| 122 |
0 |
validation_im.pyangbind_validation(self, desc_type, descriptor_data) |
| 123 |
0 |
table.append({"type": desc_type, "path": desc_path, "valid": "OK", "error": "-"}) |
| 124 |
0 |
except Exception as e: |
| 125 |
0 |
table.append({"type": desc_type, "path": desc_path, "valid": "ERROR", "error": str(e)}) |
| 126 |
0 |
return table |
| 127 |
|
|
| 128 |
1 |
def translate(self, base_directory, recursive=True, dryrun=False): |
| 129 |
|
""" |
| 130 |
|
**Translate OSM Packages given a path** |
| 131 |
|
|
| 132 |
|
:params: |
| 133 |
|
- base_directory is the root path for all packages |
| 134 |
|
|
| 135 |
|
:return: List of dict of translated packages. keys: current type, new type, path, valid, translated, error |
| 136 |
|
""" |
| 137 |
0 |
self._logger.debug("") |
| 138 |
0 |
table = [] |
| 139 |
0 |
if recursive: |
| 140 |
0 |
descriptors_paths = [f for f in glob.glob(base_directory + "/**/*.yaml", recursive=recursive)] |
| 141 |
|
else: |
| 142 |
0 |
descriptors_paths = [f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)] |
| 143 |
0 |
print("Base directory: {}".format(base_directory)) |
| 144 |
0 |
print("{} Descriptors found to validate".format(len(descriptors_paths))) |
| 145 |
0 |
for desc_path in descriptors_paths: |
| 146 |
0 |
with open(desc_path) as descriptor_file: |
| 147 |
0 |
descriptor_data = descriptor_file.read() |
| 148 |
0 |
desc_type = "-" |
| 149 |
0 |
try: |
| 150 |
0 |
desc_type, descriptor_data = validation_im.yaml_validation(self, descriptor_data) |
| 151 |
0 |
self._logger.debug("desc_type: {}".format(desc_type)) |
| 152 |
0 |
self._logger.debug("descriptor_data:\n{}".format(descriptor_data)) |
| 153 |
0 |
self._validator.pyangbind_validation(desc_type, descriptor_data) |
| 154 |
0 |
if not ( desc_type=="vnfd" or desc_type=="nsd" ): |
| 155 |
0 |
table.append({"current type": desc_type, "new type": desc_type, "path": desc_path, "valid": "OK", "translated": "N/A", "error": "-"}) |
| 156 |
|
else: |
| 157 |
0 |
new_desc_type = desc_type |
| 158 |
0 |
try: |
| 159 |
0 |
sol006_model = yaml.safe_dump(im_translation.translate_im_model_to_sol006(descriptor_data), indent=4, default_flow_style=False) |
| 160 |
0 |
new_desc_type, new_descriptor_data = self._validator.yaml_validation(sol006_model) |
| 161 |
0 |
self._validator.pyangbind_validation(new_desc_type, new_descriptor_data) |
| 162 |
0 |
if not dryrun: |
| 163 |
0 |
with open(desc_path, 'w') as descriptor_file: |
| 164 |
0 |
descriptor_file.write(sol006_model) |
| 165 |
0 |
table.append({"current type": desc_type, "new type": new_desc_type, "path": desc_path, "valid": "OK", "translated": "OK", "error": "-"}) |
| 166 |
0 |
except ValidationException as ve2: |
| 167 |
0 |
table.append({"current type": desc_type, "new type": new_desc_type, "path": desc_path, "valid": "OK", "translated": "ERROR", "error": "Error in the post-validation: {}".format(str(ve2))}) |
| 168 |
0 |
except Exception as e2: |
| 169 |
0 |
table.append({"current type": desc_type, "new type": new_desc_type, "path": desc_path, "valid": "OK", "translated": "ERROR", "error": "Error in the translation: {}".format(str(e2))}) |
| 170 |
0 |
except ValidationException as ve: |
| 171 |
0 |
table.append({"current type": desc_type, "new type": "N/A", "path": desc_path, "valid": "ERROR", "translated": "N/A", "error": "Error in the pre-validation: {}".format(str(ve))}) |
| 172 |
0 |
except Exception as e: |
| 173 |
0 |
table.append({"current type": desc_type, "new type": "N/A", "path": desc_path, "valid": "ERROR", "translated": "N/A", "error": str(e)}) |
| 174 |
0 |
return table |
| 175 |
|
|
| 176 |
1 |
def descriptor_translate(self, descriptor_file): |
| 177 |
|
""" |
| 178 |
|
**Translate input descriptor file from Rel EIGHT OSM to SOL006** |
| 179 |
|
|
| 180 |
|
:params: |
| 181 |
|
- base_directory is the root path for all packages |
| 182 |
|
|
| 183 |
|
:return: YAML descriptor in the new format |
| 184 |
|
""" |
| 185 |
0 |
self._logger.debug("") |
| 186 |
0 |
with open(descriptor_file, 'r') as df: |
| 187 |
0 |
im_model = yaml.safe_load(df.read()) |
| 188 |
0 |
sol006_model = im_translation.translate_im_model_to_sol006(im_model) |
| 189 |
0 |
return yaml.safe_dump(sol006_model, indent=4, default_flow_style=False) |
| 190 |
|
|
| 191 |
1 |
def build(self, package_folder, skip_validation=False, skip_charm_build=False): |
| 192 |
|
""" |
| 193 |
|
**Creates a .tar.gz file given a package_folder** |
| 194 |
|
|
| 195 |
|
:params: |
| 196 |
|
- package_folder: is the name of the folder to be packaged |
| 197 |
|
- skip_validation: is the flag to validate or not the descriptors on the folder before build |
| 198 |
|
|
| 199 |
|
:returns: message result for the build process |
| 200 |
|
""" |
| 201 |
0 |
self._logger.debug("") |
| 202 |
0 |
package_folder = package_folder.rstrip('/') |
| 203 |
0 |
if not os.path.exists("{}".format(package_folder)): |
| 204 |
0 |
return "Fail, package is not in the specified path" |
| 205 |
0 |
if not skip_validation: |
| 206 |
0 |
print('Validating package {}'.format(package_folder)) |
| 207 |
0 |
results = self.validate(package_folder, recursive=False) |
| 208 |
0 |
if results: |
| 209 |
0 |
for result in results: |
| 210 |
0 |
if result["valid"] != "OK": |
| 211 |
0 |
raise ClientException("There was an error validating the file {} with error: {}" |
| 212 |
|
.format(result["path"], result["error"])) |
| 213 |
0 |
print('Validation OK') |
| 214 |
|
else: |
| 215 |
0 |
raise ClientException("No descriptor file found in: {}".format(package_folder)) |
| 216 |
0 |
charm_list = self.build_all_charms(package_folder, skip_charm_build) |
| 217 |
0 |
return self.build_tarfile(package_folder, charm_list) |
| 218 |
|
|
| 219 |
1 |
def calculate_checksum(self, package_folder): |
| 220 |
|
""" |
| 221 |
|
**Function to calculate the checksum given a folder** |
| 222 |
|
|
| 223 |
|
:params: |
| 224 |
|
- package_folder: is the folder where we have the files to calculate the checksum |
| 225 |
|
:returns: None |
| 226 |
|
""" |
| 227 |
0 |
self._logger.debug("") |
| 228 |
0 |
files = [f for f in glob.glob(package_folder + "/**/*.*", recursive=True) if os.path.isfile(f)] |
| 229 |
0 |
with open("{}/checksums.txt".format(package_folder), "w+") as checksum: |
| 230 |
0 |
for file_item in files: |
| 231 |
0 |
if "checksums.txt" in file_item: |
| 232 |
0 |
continue |
| 233 |
|
# from https://www.quickprogrammingtips.com/python/how-to-calculate-md5-hash-of-a-file-in-python.html |
| 234 |
0 |
md5_hash = hashlib.md5() |
| 235 |
0 |
with open(file_item, "rb") as f: |
| 236 |
|
# Read and update hash in chunks of 4K |
| 237 |
0 |
for byte_block in iter(lambda: f.read(4096), b""): |
| 238 |
0 |
md5_hash.update(byte_block) |
| 239 |
0 |
checksum.write("{}\t{}\n".format(md5_hash.hexdigest(), file_item)) |
| 240 |
|
|
| 241 |
1 |
def create_folders(self, folders, package_type): |
| 242 |
|
""" |
| 243 |
|
**Create folder given a list of folders** |
| 244 |
|
|
| 245 |
|
:params: |
| 246 |
|
- folders: [List] list of folders paths to be created |
| 247 |
|
- package_type: is the type of package to be created |
| 248 |
|
:return: None |
| 249 |
|
""" |
| 250 |
0 |
self._logger.debug("") |
| 251 |
0 |
for folder in folders: |
| 252 |
0 |
try: |
| 253 |
|
# print("Folder {} == package_type {}".format(folder[1], package_type)) |
| 254 |
0 |
if folder[1] == package_type: |
| 255 |
0 |
print("Creating folder:\t{}".format(folder[0])) |
| 256 |
0 |
os.makedirs(folder[0]) |
| 257 |
0 |
except FileExistsError: |
| 258 |
0 |
pass |
| 259 |
|
|
| 260 |
1 |
def save_file(self, file_name, file_body): |
| 261 |
|
""" |
| 262 |
|
**Create a file given a name and the content** |
| 263 |
|
|
| 264 |
|
:params: |
| 265 |
|
- file_name: is the name of the file with the relative route |
| 266 |
|
- file_body: is the content of the file |
| 267 |
|
:return: None |
| 268 |
|
""" |
| 269 |
0 |
self._logger.debug("") |
| 270 |
0 |
print("Creating file: \t{}".format(file_name)) |
| 271 |
0 |
try: |
| 272 |
0 |
with open(file_name, "w+") as f: |
| 273 |
0 |
f.write(file_body) |
| 274 |
0 |
except Exception as e: |
| 275 |
0 |
raise ClientException(e) |
| 276 |
|
|
| 277 |
1 |
def generate_readme(self): |
| 278 |
|
""" |
| 279 |
|
**Creates the README content** |
| 280 |
|
|
| 281 |
|
:returns: readme content |
| 282 |
|
""" |
| 283 |
0 |
self._logger.debug("") |
| 284 |
0 |
return """# Descriptor created by OSM descriptor package generated\n\n**Created on {} **""".format( |
| 285 |
|
time.strftime("%m/%d/%Y, %H:%M:%S", time.localtime())) |
| 286 |
|
|
| 287 |
1 |
def generate_cloud_init(self): |
| 288 |
|
""" |
| 289 |
|
**Creates the cloud-init content** |
| 290 |
|
|
| 291 |
|
:returns: cloud-init content |
| 292 |
|
""" |
| 293 |
0 |
self._logger.debug("") |
| 294 |
0 |
return "---\n#cloud-config" |
| 295 |
|
|
| 296 |
1 |
def create_files(self, files, file_content, package_type): |
| 297 |
|
""" |
| 298 |
|
**Creates the files given the file list and type** |
| 299 |
|
|
| 300 |
|
:params: |
| 301 |
|
- files: is the list of files structure |
| 302 |
|
- file_content: is the content of the descriptor rendered by the template |
| 303 |
|
- package_type: is the type of package to filter the creation structure |
| 304 |
|
|
| 305 |
|
:return: None |
| 306 |
|
""" |
| 307 |
0 |
self._logger.debug("") |
| 308 |
0 |
for file_item, file_package, file_type in files: |
| 309 |
0 |
if package_type == file_package: |
| 310 |
0 |
if file_type == "descriptor": |
| 311 |
0 |
self.save_file(file_item, file_content) |
| 312 |
0 |
elif file_type == "readme": |
| 313 |
0 |
self.save_file(file_item, self.generate_readme()) |
| 314 |
0 |
elif file_type == "cloud_init": |
| 315 |
0 |
self.save_file(file_item, self.generate_cloud_init()) |
| 316 |
|
|
| 317 |
1 |
def check_files_folders(self, path_list, override): |
| 318 |
|
""" |
| 319 |
|
**Find files and folders missing given a directory structure {"folders": [], "files": []}** |
| 320 |
|
|
| 321 |
|
:params: |
| 322 |
|
- path_list: is the list of files and folders to be created |
| 323 |
|
- override: is the flag used to indicate the creation of the list even if the file exist to override it |
| 324 |
|
|
| 325 |
|
:return: Missing paths Dict |
| 326 |
|
""" |
| 327 |
0 |
self._logger.debug("") |
| 328 |
0 |
missing_paths = {} |
| 329 |
0 |
folders = [] |
| 330 |
0 |
files = [] |
| 331 |
0 |
for folder in path_list.get("folders"): |
| 332 |
0 |
if not os.path.exists(folder[0]): |
| 333 |
0 |
folders.append(folder) |
| 334 |
0 |
missing_paths["folders"] = folders |
| 335 |
|
|
| 336 |
0 |
for file_item in path_list.get("files"): |
| 337 |
0 |
if not os.path.exists(file_item[0]) or override is True: |
| 338 |
0 |
files.append(file_item) |
| 339 |
0 |
missing_paths["files"] = files |
| 340 |
|
|
| 341 |
0 |
return missing_paths |
| 342 |
|
|
| 343 |
1 |
def build_all_charms(self, package_folder, skip_charm_build): |
| 344 |
|
""" |
| 345 |
|
**Read the descriptor file, check that the charms referenced are in the folder and compiles them** |
| 346 |
|
|
| 347 |
|
:params: |
| 348 |
|
- packet_folder: is the location of the package |
| 349 |
|
:return: Files and Folders not found. In case of override, it will return all file list |
| 350 |
|
""" |
| 351 |
0 |
self._logger.debug("") |
| 352 |
0 |
listCharms = [] |
| 353 |
0 |
descriptor_file = False |
| 354 |
0 |
descriptors_paths = [f for f in glob.glob(package_folder + "/*.yaml")] |
| 355 |
0 |
for file in descriptors_paths: |
| 356 |
0 |
if file.endswith('nfd.yaml'): |
| 357 |
0 |
descriptor_file = True |
| 358 |
0 |
listCharms = self.charms_search(file, 'vnf') |
| 359 |
0 |
if file.endswith('nsd.yaml'): |
| 360 |
0 |
descriptor_file = True |
| 361 |
0 |
listCharms = self.charms_search(file, 'ns') |
| 362 |
0 |
print("List of charms in the descriptor: {}".format(listCharms)) |
| 363 |
0 |
if not descriptor_file: |
| 364 |
0 |
raise ClientException('Descriptor filename is not correct in: {}. It should end with "nfd.yaml" or "nsd.yaml"'.format(package_folder)) |
| 365 |
0 |
if listCharms and not skip_charm_build: |
| 366 |
0 |
for charmName in listCharms: |
| 367 |
0 |
if os.path.isdir('{}/charms/layers/{}'.format(package_folder, charmName)): |
| 368 |
0 |
print('Building charm {}/charms/layers/{}'.format(package_folder, charmName)) |
| 369 |
0 |
self.charm_build(package_folder, charmName) |
| 370 |
0 |
print('Charm built: {}'.format(charmName)) |
| 371 |
|
else: |
| 372 |
0 |
if not os.path.isdir('{}/charms/{}'.format(package_folder, charmName)): |
| 373 |
0 |
raise ClientException('The charm: {} referenced in the descriptor file ' |
| 374 |
|
'is not present either in {}/charms or in {}/charms/layers'. |
| 375 |
|
format(charmName, package_folder, package_folder)) |
| 376 |
0 |
self._logger.debug("Return list of charms: {}".format(listCharms)) |
| 377 |
0 |
return listCharms |
| 378 |
|
|
| 379 |
1 |
def discover_folder_structure(self, base_directory, name, override): |
| 380 |
|
""" |
| 381 |
|
**Discover files and folders structure for OSM descriptors given a base_directory and name** |
| 382 |
|
|
| 383 |
|
:params: |
| 384 |
|
- base_directory: is the location of the package to be created |
| 385 |
|
- name: is the name of the package |
| 386 |
|
- override: is the flag used to indicate the creation of the list even if the file exist to override it |
| 387 |
|
:return: Files and Folders not found. In case of override, it will return all file list |
| 388 |
|
""" |
| 389 |
0 |
self._logger.debug("") |
| 390 |
0 |
prefix = "{}/{}".format(base_directory, name) |
| 391 |
0 |
files_folders = {"folders": [("{}_ns".format(prefix), "ns"), |
| 392 |
|
("{}_ns/icons".format(prefix), "ns"), |
| 393 |
|
("{}_ns/charms".format(prefix), "ns"), |
| 394 |
|
("{}_vnf".format(name), "vnf"), |
| 395 |
|
("{}_vnf/charms".format(prefix), "vnf"), |
| 396 |
|
("{}_vnf/cloud_init".format(prefix), "vnf"), |
| 397 |
|
("{}_vnf/images".format(prefix), "vnf"), |
| 398 |
|
("{}_vnf/icons".format(prefix), "vnf"), |
| 399 |
|
("{}_vnf/scripts".format(prefix), "vnf"), |
| 400 |
|
("{}_nst".format(prefix), "nst"), |
| 401 |
|
("{}_nst/icons".format(prefix), "nst") |
| 402 |
|
], |
| 403 |
|
"files": [("{}_ns/{}_nsd.yaml".format(prefix, name), "ns", "descriptor"), |
| 404 |
|
("{}_ns/README.md".format(prefix), "ns", "readme"), |
| 405 |
|
("{}_vnf/{}_vnfd.yaml".format(prefix, name), "vnf", "descriptor"), |
| 406 |
|
("{}_vnf/cloud_init/cloud-config.txt".format(prefix), "vnf", "cloud_init"), |
| 407 |
|
("{}_vnf/README.md".format(prefix), "vnf", "readme"), |
| 408 |
|
("{}_nst/{}_nst.yaml".format(prefix, name), "nst", "descriptor"), |
| 409 |
|
("{}_nst/README.md".format(prefix), "nst", "readme") |
| 410 |
|
] |
| 411 |
|
} |
| 412 |
0 |
missing_files_folders = self.check_files_folders(files_folders, override) |
| 413 |
|
# print("Missing files and folders: {}".format(missing_files_folders)) |
| 414 |
0 |
return missing_files_folders |
| 415 |
|
|
| 416 |
1 |
def charm_build(self, charms_folder, build_name): |
| 417 |
|
""" |
| 418 |
|
Build the charms inside the package. |
| 419 |
|
params: package_folder is the name of the folder where is the charms to compile. |
| 420 |
|
build_name is the name of the layer or interface |
| 421 |
|
""" |
| 422 |
0 |
self._logger.debug("") |
| 423 |
0 |
os.environ['JUJU_REPOSITORY'] = "{}/charms".format(charms_folder) |
| 424 |
0 |
os.environ['CHARM_LAYERS_DIR'] = "{}/layers".format(os.environ['JUJU_REPOSITORY']) |
| 425 |
0 |
os.environ['CHARM_INTERFACES_DIR'] = "{}/interfaces".format(os.environ['JUJU_REPOSITORY']) |
| 426 |
0 |
os.environ['CHARM_BUILD_DIR'] = "{}/charms/builds".format(charms_folder) |
| 427 |
0 |
if not os.path.exists(os.environ['CHARM_BUILD_DIR']): |
| 428 |
0 |
os.makedirs(os.environ['CHARM_BUILD_DIR']) |
| 429 |
0 |
src_folder = '{}/{}'.format(os.environ['CHARM_LAYERS_DIR'], build_name) |
| 430 |
0 |
result = subprocess.run(["charm", "build", "{}".format(src_folder)]) |
| 431 |
0 |
if result.returncode == 1: |
| 432 |
0 |
raise ClientException("failed to build the charm: {}".format(src_folder)) |
| 433 |
0 |
self._logger.verbose("charm {} built".format(src_folder)) |
| 434 |
|
|
| 435 |
1 |
def build_tarfile(self, package_folder, charm_list=None): |
| 436 |
|
""" |
| 437 |
|
Creates a .tar.gz file given a package_folder |
| 438 |
|
params: package_folder is the name of the folder to be packaged |
| 439 |
|
returns: .tar.gz name |
| 440 |
|
""" |
| 441 |
0 |
self._logger.debug("") |
| 442 |
0 |
cwd = None |
| 443 |
0 |
try: |
| 444 |
0 |
directory_name, package_name = self.create_temp_dir(package_folder, charm_list) |
| 445 |
0 |
cwd = os.getcwd() |
| 446 |
0 |
os.chdir(directory_name) |
| 447 |
0 |
self.calculate_checksum(package_name) |
| 448 |
0 |
with tarfile.open("{}.tar.gz".format(package_name), mode='w:gz') as archive: |
| 449 |
0 |
print("Adding File: {}".format(package_name)) |
| 450 |
0 |
archive.add('{}'.format(package_name), recursive=True) |
| 451 |
|
# return "Created {}.tar.gz".format(package_folder) |
| 452 |
|
# self.build("{}".format(os.path.basename(package_folder))) |
| 453 |
0 |
os.chdir(cwd) |
| 454 |
0 |
cwd = None |
| 455 |
0 |
created_package = "{}/{}.tar.gz".format(os.path.dirname(package_folder) or '.', package_name) |
| 456 |
0 |
os.rename("{}/{}.tar.gz".format(directory_name, package_name), |
| 457 |
|
created_package) |
| 458 |
0 |
os.rename("{}/{}/checksums.txt".format(directory_name, package_name), |
| 459 |
|
"{}/checksums.txt".format(package_folder)) |
| 460 |
0 |
print("Package created: {}".format(created_package)) |
| 461 |
0 |
return created_package |
| 462 |
0 |
except Exception as exc: |
| 463 |
0 |
raise ClientException('failure during build of targz file (create temp dir, calculate checksum, ' |
| 464 |
|
'tar.gz file): {}'.format(exc)) |
| 465 |
|
finally: |
| 466 |
0 |
if cwd: |
| 467 |
0 |
os.chdir(cwd) |
| 468 |
0 |
shutil.rmtree(os.path.join(package_folder, "tmp")) |
| 469 |
|
|
| 470 |
1 |
def create_temp_dir(self, package_folder, charm_list=None): |
| 471 |
|
""" |
| 472 |
|
Method to create a temporary folder where we can move the files in package_folder |
| 473 |
|
""" |
| 474 |
0 |
self._logger.debug("") |
| 475 |
0 |
ignore_patterns = ('.gitignore') |
| 476 |
0 |
ignore = shutil.ignore_patterns(ignore_patterns) |
| 477 |
0 |
directory_name = os.path.abspath(package_folder) |
| 478 |
0 |
package_name = os.path.basename(directory_name) |
| 479 |
0 |
directory_name += "/tmp" |
| 480 |
0 |
os.makedirs("{}/{}".format(directory_name, package_name), exist_ok=True) |
| 481 |
0 |
self._logger.debug("Makedirs DONE: {}/{}".format(directory_name, package_name)) |
| 482 |
0 |
for item in os.listdir(package_folder): |
| 483 |
0 |
self._logger.debug("Item: {}".format(item)) |
| 484 |
0 |
if item != "tmp": |
| 485 |
0 |
s = os.path.join(package_folder, item) |
| 486 |
0 |
d = os.path.join(os.path.join(directory_name, package_name), item) |
| 487 |
0 |
if os.path.isdir(s): |
| 488 |
0 |
if item == "charms": |
| 489 |
0 |
os.makedirs(d, exist_ok=True) |
| 490 |
0 |
s_builds = os.path.join(s, "builds") |
| 491 |
0 |
for charm in charm_list: |
| 492 |
0 |
self._logger.debug("Copying charm {}".format(charm)) |
| 493 |
0 |
if charm in os.listdir(s): |
| 494 |
0 |
s_charm = os.path.join(s, charm) |
| 495 |
0 |
elif charm in os.listdir(s_builds): |
| 496 |
0 |
s_charm = os.path.join(s_builds, charm) |
| 497 |
|
else: |
| 498 |
0 |
raise ClientException('The charm {} referenced in the descriptor file ' |
| 499 |
|
'could not be found in {}/charms or in {}/charms/builds'. |
| 500 |
|
format(charm, package_folder, package_folder)) |
| 501 |
0 |
d_temp = os.path.join(d, charm) |
| 502 |
0 |
self._logger.debug("Copying tree: {} -> {}".format(s_charm, d_temp)) |
| 503 |
0 |
shutil.copytree(s_charm, d_temp, symlinks=True, ignore=ignore) |
| 504 |
0 |
self._logger.debug("DONE") |
| 505 |
|
else: |
| 506 |
0 |
self._logger.debug("Copying tree: {} -> {}".format(s, d)) |
| 507 |
0 |
shutil.copytree(s, d, symlinks=True, ignore=ignore) |
| 508 |
0 |
self._logger.debug("DONE") |
| 509 |
|
else: |
| 510 |
0 |
if item in ignore_patterns: |
| 511 |
0 |
continue |
| 512 |
0 |
self._logger.debug("Copying file: {} -> {}".format(s, d)) |
| 513 |
0 |
shutil.copy2(s, d) |
| 514 |
0 |
self._logger.debug("DONE") |
| 515 |
0 |
return directory_name, package_name |
| 516 |
|
|
| 517 |
1 |
def charms_search(self, descriptor_file, desc_type): |
| 518 |
0 |
self._logger.debug("descriptor_file: {}, desc_type: {}".format(descriptor_file, |
| 519 |
|
desc_type)) |
| 520 |
0 |
with open("{}".format(descriptor_file)) as yaml_desc: |
| 521 |
0 |
descriptor_dict = yaml.safe_load(yaml_desc) |
| 522 |
|
#self._logger.debug("\n"+yaml.safe_dump(descriptor_dict, indent=4, default_flow_style=False)) |
| 523 |
|
|
| 524 |
0 |
if ( (desc_type=="vnf" and ("vnfd:vnfd-catalog" in descriptor_dict or "vnfd-catalog" in descriptor_dict)) or |
| 525 |
|
(desc_type=="ns" and ( "nsd:nsd-catalog" in descriptor_dict or "nsd-catalog" in descriptor_dict)) ): |
| 526 |
0 |
charms_list = self._charms_search_on_osm_im_dict(descriptor_dict, desc_type) |
| 527 |
|
else: |
| 528 |
0 |
if desc_type == "ns": |
| 529 |
0 |
get_charm_list = self._charms_search_on_nsd_sol006_dict |
| 530 |
0 |
elif desc_type == "vnf": |
| 531 |
0 |
get_charm_list = self._charms_search_on_vnfd_sol006_dict |
| 532 |
|
else: |
| 533 |
0 |
raise Exception("Bad descriptor type") |
| 534 |
0 |
charms_list = get_charm_list(descriptor_dict) |
| 535 |
0 |
return charms_list |
| 536 |
|
|
| 537 |
1 |
def _charms_search_on_osm_im_dict(self, osm_im_dict, desc_type): |
| 538 |
0 |
self._logger.debug("") |
| 539 |
0 |
charms_list = [] |
| 540 |
0 |
for k1, v1 in osm_im_dict.items(): |
| 541 |
0 |
for k2, v2 in v1.items(): |
| 542 |
0 |
for entry in v2: |
| 543 |
0 |
if '{}-configuration'.format(desc_type) in entry: |
| 544 |
0 |
vnf_config = entry['{}-configuration'.format(desc_type)] |
| 545 |
0 |
for k3, v3 in vnf_config.items(): |
| 546 |
0 |
if 'charm' in v3: |
| 547 |
0 |
charms_list.append((v3['charm'])) |
| 548 |
0 |
if 'vdu' in entry: |
| 549 |
0 |
vdus = entry['vdu'] |
| 550 |
0 |
for vdu in vdus: |
| 551 |
0 |
if 'vdu-configuration' in vdu: |
| 552 |
0 |
for k4, v4 in vdu['vdu-configuration'].items(): |
| 553 |
0 |
if 'charm' in v4: |
| 554 |
0 |
charms_list.append((v4['charm'])) |
| 555 |
0 |
return charms_list |
| 556 |
|
|
| 557 |
1 |
def _charms_search_on_vnfd_sol006_dict(self, sol006_dict): |
| 558 |
0 |
self._logger.debug("") |
| 559 |
0 |
charms_list = [] |
| 560 |
0 |
for k1, v1 in sol006_dict.items(): |
| 561 |
0 |
for k2, v2 in v1.items(): |
| 562 |
0 |
if k2 == "df": |
| 563 |
0 |
for df in v2: |
| 564 |
0 |
lcm_ops = df.get("lcm-operations-configuration", {}) |
| 565 |
0 |
ops_config = lcm_ops.get("operate-vnf-op-config", {}) |
| 566 |
0 |
for day_12_config in ops_config.get("day1-2", []): |
| 567 |
0 |
self._logger.debug("Execution environment found") |
| 568 |
0 |
for ee in day_12_config.get("execution-environment-list", []): |
| 569 |
0 |
if "juju" in ee: |
| 570 |
0 |
charms_list.append((ee["juju"]['charm'])) |
| 571 |
0 |
return charms_list |
| 572 |
|
|
| 573 |
1 |
def _charms_search_on_nsd_sol006_dict(self, sol006_dict): |
| 574 |
0 |
self._logger.debug("") |
| 575 |
0 |
charms_list = [] |
| 576 |
0 |
nsd_list = sol006_dict.get("nsd", {}).get("nsd", []) |
| 577 |
0 |
for nsd in nsd_list: |
| 578 |
0 |
charm = nsd.get("ns-configuration", {}).get("juju", {}).get("charm") |
| 579 |
0 |
if charm: |
| 580 |
0 |
charms_list.append(charm) |
| 581 |
0 |
return charms_list |
![[Jenkins]](/jenkins/static/c06f6c43/images/jenkins-header-logo-v2.svg){kind=logo}
