2 # Licensed under the Apache License, Version 2.0 (the "License"); you may
3 # not use this file except in compliance with the License. You may obtain
4 # a copy of the License at
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
10 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
11 # License for the specific language governing permissions and limitations
21 from os
import listdir
, mkdir
, getcwd
, remove
22 from os
.path
import isfile
, isdir
, join
, abspath
23 from shutil
import copyfile
, rmtree
28 from osm_im
.validation
import Validation
as validation_im
29 from osmclient
.common
.exceptions
import ClientException
30 from osmclient
.common
.package_tool
import PackageTool
31 from osmclient
.sol005
.repo
import Repo
32 from packaging
import version
as versioning
38 def __init__(self
, http
=None, client
=None):
41 self
._apiName
= '/admin'
42 self
._apiVersion
= '/v1'
43 self
._apiResource
= '/osmrepos'
44 self
._logger
= logging
.getLogger('osmclient')
45 self
._apiBase
= '{}{}{}'.format(self
._apiName
,
46 self
._apiVersion
, self
._apiResource
)
48 def pkg_list(self
, pkgtype
, filter=None, repo
=None):
50 Returns a repo based on name or id
52 self
._logger
.debug("")
53 self
._client
.get_token()
54 # Get OSM registered repository list
55 repositories
= self
.list()
57 repositories
= [r
for r
in repositories
if r
["name"] == repo
]
59 raise ClientException('Not repository found')
62 for repository
in repositories
:
64 r
= requests
.get('{}/index.yaml'.format(repository
.get('url')))
66 if r
.status_code
== 200:
67 repo_list
= yaml
.safe_load(r
.text
)
68 vnf_packages
= repo_list
.get('{}_packages'.format(pkgtype
))
69 for repo
in vnf_packages
:
70 versions
= vnf_packages
.get(repo
)
71 latest
= versions
.get('latest')
72 del versions
['latest']
73 for version
in versions
:
74 latest_version
= False
77 vnf_repos
.append({'vendor': versions
[version
].get("vendor"),
78 'name': versions
[version
].get("name"),
80 'description': versions
[version
].get("description"),
81 'location': versions
[version
].get("path"),
82 'repository': repository
.get('name'),
83 'repourl': repository
.get('url'),
84 'latest': latest_version
87 raise Exception('repository in url {} unreachable'.format(repository
.get('url')))
88 except Exception as e
:
90 "Error cannot read from repository {} '{}': {}".format(
91 repository
["name"], repository
["url"], e
97 vnf_repos_filtered
= []
99 for vnf_repo
in vnf_repos
:
100 for k
, v
in vnf_repo
.items():
102 kf
, vf
= filter.split('=')
103 if k
== kf
and vf
in v
:
104 vnf_repos_filtered
.append(vnf_repo
)
106 vnf_repos
= vnf_repos_filtered
109 def get_pkg(self
, pkgtype
, name
, repo
, filter, version
):
111 Returns the filename of the PKG downloaded to disk
113 self
._logger
.debug("")
114 self
._client
.get_token()
117 # Get OSM registered repository list
118 pkgs
= self
.pkg_list(pkgtype
, filter, repo
)
120 if pkg
.get('repository') == repo
and pkg
.get('name') == name
:
121 if 'latest' in version
:
122 if not pkg
.get('latest'):
125 version
= pkg
.get('version')
126 if pkg
.get('version') == version
:
127 r
= requests
.get('{}{}'.format(pkg
.get('repourl'), pkg
.get('location')), stream
=True)
128 if r
.status_code
!= 200:
129 raise ClientException("Package not found")
131 with tempfile
.NamedTemporaryFile(delete
=False) as f
:
132 f
.write(r
.raw
.read())
135 raise ClientException("{} {} not found at repo {}".format(pkgtype
, name
, repo
))
138 def pkg_get(self
, pkgtype
, name
, repo
, version
, filter):
140 pkg_name
= self
.get_pkg(pkgtype
, name
, repo
, filter, version
)
142 raise ClientException('Package not found')
143 folder
, descriptor
= self
.zip_extraction(pkg_name
)
144 with
open(descriptor
) as pkg
:
145 pkg_descriptor
= yaml
.safe_load(pkg
)
146 rmtree(folder
, ignore_errors
=False)
147 if ((pkgtype
== 'vnf' and (pkg_descriptor
.get('vnfd') or pkg_descriptor
.get('vnfd:vnfd_catalog'))) or
148 (pkgtype
== 'ns' and (pkg_descriptor
.get('nsd') or pkg_descriptor
.get('nsd:nsd_catalog')))):
149 raise ClientException('Wrong Package type')
150 return pkg_descriptor
152 def repo_index(self
, origin
=".", destination
='.'):
154 Repo Index main function
155 :param origin: origin directory for getting all the artifacts
156 :param destination: destination folder for create and index the valid artifacts
158 self
._logger
.debug("Starting index composition")
159 if destination
== ".":
160 if origin
== destination
:
161 destination
= 'repository'
163 destination
= abspath(destination
)
164 origin
= abspath(origin
)
165 self
._logger
.debug(f
"Paths {destination}, {origin}")
167 origin
= join(getcwd(), origin
)
168 if destination
[0] != '/':
169 destination
= join(getcwd(), destination
)
171 self
.init_directory(destination
)
174 for f
in listdir(origin
):
175 if isfile(join(origin
, f
)) and f
.endswith('.tar.gz'):
177 elif isdir(join(origin
, f
)) and f
!= destination
.split('/')[-1] and not f
.startswith('.'):
178 directories
.append(f
) # TODO: Document that nested directories are not supported
180 self
._logger
.debug(f
"Ignoring {f}")
181 for artifact
in artifacts
:
182 self
.register_artifact_in_repository(
183 join(origin
, artifact
), destination
, source
="artifact"
185 for artifact
in directories
:
186 self
.register_artifact_in_repository(
187 join(origin
, artifact
), destination
, source
="directory"
189 self
._logger
.info("\nFinal Results: ")
191 "VNF Packages Indexed: "
192 + str(len(glob
.glob(destination
+ "/vnf/*/*/metadata.yaml")))
195 "NS Packages Indexed: "
196 + str(len(glob
.glob(destination
+ "/ns/*/*/metadata.yaml")))
200 "NST Packages Indexed: "
201 + str(len(glob
.glob(destination
+ "/nst/*/*/metadata.yaml")))
204 def md5(self
, fname
):
207 :param fname: file path
208 :return: checksum string
210 self
._logger
.debug("")
211 hash_md5
= hashlib
.md5()
212 with
open(fname
, "rb") as f
:
213 for chunk
in iter(lambda: f
.read(4096), b
""):
214 hash_md5
.update(chunk
)
215 return hash_md5
.hexdigest()
217 def fields_building(self
, descriptor_dict
, file, package_type
):
219 From an artifact descriptor, obtain the fields required for indexing
220 :param descriptor_dict: artifact description
221 :param file: artifact package
222 :param package_type: type of artifact (vnf, ns, nst)
225 self
._logger
.debug("")
228 base_path
= '/{}/'.format(package_type
)
230 if package_type
== "vnf":
231 if descriptor_dict
.get("vnfd-catalog", False):
232 aux_dict
= descriptor_dict
.get("vnfd-catalog", {}).get("vnfd", [{}])[0]
233 elif descriptor_dict
.get("vnfd:vnfd-catalog"):
234 aux_dict
= descriptor_dict
.get("vnfd:vnfd-catalog", {}).get("vnfd", [{}])[0]
235 elif descriptor_dict
.get("vnfd"):
236 aux_dict
= descriptor_dict
["vnfd"]
237 if aux_dict
.get("vnfd"):
238 aux_dict
= aux_dict
['vnfd'][0]
240 msg
= f
"Unexpected descriptor format {descriptor_dict}"
241 self
._logger
.error(msg
)
242 raise ValueError(msg
)
243 self
._logger
.debug(f
"Extracted descriptor info for {package_type}: {aux_dict}")
245 for vdu
in aux_dict
.get("vdu", aux_dict
.get('kdu', ())):
246 images
.append(vdu
.get("image", vdu
.get('name')))
247 fields
["images"] = images
248 elif package_type
== "ns":
249 if descriptor_dict
.get("nsd-catalog", False):
250 aux_dict
= descriptor_dict
.get("nsd-catalog", {}).get("nsd", [{}])[0]
251 elif descriptor_dict
.get("nsd:nsd-catalog"):
252 aux_dict
= descriptor_dict
.get("nsd:nsd-catalog", {}).get("nsd", [{}])[0]
253 elif descriptor_dict
.get("nsd"):
254 aux_dict
= descriptor_dict
['nsd']
255 if aux_dict
.get("nsd"):
256 aux_dict
= descriptor_dict
["nsd"]["nsd"][0]
258 msg
= f
"Unexpected descriptor format {descriptor_dict}"
259 self
._logger
.error(msg
)
260 raise ValueError(msg
)
262 if aux_dict
.get("constituent-vnfd"):
263 for vnf
in aux_dict
.get("constituent-vnfd", ()):
264 vnfs
.append(vnf
.get("vnfd-id-ref"))
266 vnfs
= aux_dict
.get('vnfd-id')
267 self
._logger
.debug("Used VNFS in the NSD: " + str(vnfs
))
268 fields
["vnfd-id-ref"] = vnfs
269 elif package_type
== 'nst':
270 if descriptor_dict
.get("nst-catalog", False):
271 aux_dict
= descriptor_dict
.get("nst-catalog", {}).get("nst", [{}])[0]
272 elif descriptor_dict
.get("nst:nst-catalog"):
273 aux_dict
= descriptor_dict
.get("nst:nst-catalog", {}).get("nst", [{}])[0]
274 elif descriptor_dict
.get("nst"):
275 aux_dict
= descriptor_dict
['nst']
276 if aux_dict
.get("nst"):
277 aux_dict
= descriptor_dict
["nst"]["nst"][0]
279 for nsd
in aux_dict
.get("netslice-subnet", ()):
280 nsds
.append(nsd
.get("nsd-ref"))
281 self
._logger
.debug("Used NSDs in the NST: " + str(nsds
))
283 msg
= f
"Unexpected descriptor format {descriptor_dict}"
284 self
._logger
.error(msg
)
285 raise ValueError(msg
)
286 fields
["nsd-id-ref"] = nsds
288 msg
= f
"Unexpected descriptor format {descriptor_dict}"
289 self
._logger
.error(msg
)
290 raise ValueError(msg
)
292 fields
["name"] = aux_dict
.get("name")
293 fields
["id"] = aux_dict
.get("id")
294 fields
["description"] = aux_dict
.get("description")
295 fields
["vendor"] = aux_dict
.get("vendor")
296 fields
["version"] = str(aux_dict
.get("version", "1.0"))
297 fields
["path"] = "{}{}/{}/{}-{}.tar.gz".format(
302 fields
.get("version"),
306 def zip_extraction(self
, file_name
):
308 Validation of artifact.
309 :param file: file path
310 :return: status details, status, fields, package_type
312 self
._logger
.debug("Decompressing package file")
313 temp_file
= '/tmp/{}'.format(file_name
.split('/')[-1])
314 if file_name
!= temp_file
:
315 copyfile(file_name
, temp_file
)
316 with tarfile
.open(temp_file
, "r:gz") as tar
:
317 folder
= tar
.getnames()[0].split('/')[0]
321 descriptor_file
= glob
.glob('{}/*.y*ml'.format(folder
))[0]
322 return folder
, descriptor_file
324 def validate_artifact(self
, path
, source
):
326 Validation of artifact.
327 :param path: file path
328 :param source: flag to select the correct file type (directory or artifact)
329 :return: status details, status, fields, package_type
331 self
._logger
.debug("")
335 if source
== 'directory':
336 descriptor_file
= glob
.glob('{}/*.y*ml'.format(path
))[0]
338 folder
, descriptor_file
= self
.zip_extraction(path
)
340 self
._logger
.debug("Opening descriptor file: {}".format(descriptor_file
))
342 with
open(descriptor_file
, 'r') as f
:
343 descriptor_data
= f
.read()
344 self
._logger
.debug(f
"Descriptor data: {descriptor_data}")
345 validation
= validation_im()
346 desc_type
, descriptor_dict
= validation
.yaml_validation(descriptor_data
)
348 validation_im
.pyangbind_validation(self
, desc_type
, descriptor_dict
)
349 except Exception as e
:
350 self
._logger
.error(e
, exc_info
=True)
352 descriptor_type_ref
= list(descriptor_dict
.keys())[0].lower()
353 if "vnf" in descriptor_type_ref
:
355 elif "nst" in descriptor_type_ref
:
357 elif "ns" in descriptor_type_ref
:
360 msg
= f
"Unknown package type {descriptor_type_ref}"
361 self
._logger
.error(msg
)
362 raise ValueError(msg
)
363 self
._logger
.debug("Descriptor: {}".format(descriptor_dict
))
364 fields
= self
.fields_building(descriptor_dict
, path
, package_type
)
365 self
._logger
.debug(f
"Descriptor successfully validated {fields}")
368 "detail": "{}D successfully validated".format(package_type
.upper()),
375 except Exception as e
:
376 # Delete the folder we just created
377 return {"detail": str(e
)}, False, {}, package_type
380 rmtree(folder
, ignore_errors
=True)
382 def register_artifact_in_repository(self
, path
, destination
, source
):
384 Registration of one artifact in a repository
386 destination: path for index creation
388 self
._logger
.debug("")
393 _
, valid
, fields
, package_type
= self
.validate_artifact(path
, source
)
396 "{} {} Not well configured.".format(package_type
.upper(), str(path
))
399 if source
== "directory":
400 path
= pt
.build(path
)
401 self
._logger
.debug(f
"Directory path {path}")
403 fields
["checksum"] = self
.md5(path
)
404 self
.indexation(destination
, path
, package_type
, fields
)
406 except Exception as e
:
407 self
._logger
.exception("Error registering artifact in Repository: {}".format(e
))
410 if source
== "directory" and compressed
:
413 def indexation(self
, destination
, path
, package_type
, fields
):
415 Process for index packages
416 :param destination: index repository path
417 :param path: path of the package
418 :param package_type: package type (vnf, ns)
419 :param fields: dict with the required values
421 self
._logger
.debug("")
422 data_ind
= {'name': fields
.get('name'), 'description': fields
.get('description'),
423 'vendor': fields
.get('vendor'), 'path': fields
.get('path')}
425 final_path
= join(destination
, package_type
, fields
.get('id'), fields
.get('version'))
426 if isdir(join(destination
, package_type
, fields
.get('id'))):
427 if isdir(final_path
):
428 self
._logger
.warning('{} {} already exists'.format(package_type
.upper(), str(path
)))
432 final_path
+ '/' + fields
.get('id') + "-" + fields
.get('version') + '.tar.gz')
433 yaml
.safe_dump(fields
, open(final_path
+ '/' + 'metadata.yaml', 'w'),
434 default_flow_style
=False, width
=80, indent
=4)
435 index
= yaml
.safe_load(open(destination
+ '/index.yaml'))
437 index
['{}_packages'.format(package_type
)][fields
.get('id')][fields
.get('version')] = data_ind
438 if versioning
.parse(index
['{}_packages'.format(package_type
)][fields
.get('id')][
439 'latest']) < versioning
.parse(fields
.get('version')):
440 index
['{}_packages'.format(package_type
)][fields
.get('id')]['latest'] = fields
.get(
442 yaml
.safe_dump(index
, open(destination
+ '/index.yaml', 'w'),
443 default_flow_style
=False, width
=80, indent
=4)
444 self
._logger
.info('{} {} added in the repository'.format(package_type
.upper(), str(path
)))
446 mkdir(destination
+ '/{}/'.format(package_type
) + fields
.get('id'))
449 final_path
+ '/' + fields
.get('id') + "-" + fields
.get('version') + '.tar.gz')
450 yaml
.safe_dump(fields
, open(join(final_path
, 'metadata.yaml'), 'w'),
451 default_flow_style
=False, width
=80, indent
=4)
452 index
= yaml
.safe_load(open(destination
+ '/index.yaml'))
454 index
['{}_packages'.format(package_type
)][fields
.get('id')] = {fields
.get('version'): data_ind
}
455 index
['{}_packages'.format(package_type
)][fields
.get('id')]['latest'] = fields
.get('version')
456 yaml
.safe_dump(index
, open(join(destination
, 'index.yaml'), 'w'),
457 default_flow_style
=False, width
=80, indent
=4)
458 self
._logger
.info('{} {} added in the repository'.format(package_type
.upper(), str(path
)))
460 def current_datetime(self
):
463 :return: Datetime as string with the following structure "2020-04-29T08:41:07.681653Z"
465 self
._logger
.debug("")
466 return time
.strftime('%Y-%m-%dT%H:%M:%S.%sZ')
468 def init_directory(self
, destination
):
470 Initialize the index directory. Creation of index.yaml, and the directories for vnf and ns
474 self
._logger
.debug("")
475 if not isdir(destination
):
477 if not isfile(join(destination
, "index.yaml")):
478 mkdir(join(destination
, "vnf"))
479 mkdir(join(destination
, "ns"))
480 mkdir(join(destination
, "nst"))
483 "generated": self
.current_datetime(),
488 with
open(join(destination
, "index.yaml"), "w") as outfile
:
490 index_data
, outfile
, default_flow_style
=False, width
=80, indent
=4