2 # Licensed under the Apache License, Version 2.0 (the "License"); you may
3 # not use this file except in compliance with the License. You may obtain
4 # a copy of the License at
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
10 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
11 # License for the specific language governing permissions and limitations
18 from osmclient
.common
.exceptions
import ClientException
19 from osmclient
.sol005
.repo
import Repo
20 from osmclient
.common
.package_tool
import PackageTool
24 from shutil
import copyfile
, rmtree
28 from packaging
import version
as versioning
30 from os
import listdir
, mkdir
, getcwd
, remove
31 from os
.path
import isfile
, isdir
, join
, abspath
33 from osm_im
.validation
import Validation
as validation_im
38 def __init__(self
, http
=None, client
=None):
41 self
._apiName
= '/admin'
42 self
._apiVersion
= '/v1'
43 self
._apiResource
= '/osmrepos'
44 self
._logger
= logging
.getLogger('osmclient')
45 self
._apiBase
= '{}{}{}'.format(self
._apiName
,
46 self
._apiVersion
, self
._apiResource
)
48 def pkg_list(self
, pkgtype
, filter=None, repo
=None):
50 Returns a repo based on name or id
52 self
._logger
.debug("")
53 self
._client
.get_token()
54 # Get OSM registered repository list
55 repositories
= self
.list()
57 repositories
= [r
for r
in repositories
if r
["name"] == repo
]
59 raise ClientException('Not repository found')
62 for repository
in repositories
:
64 r
= requests
.get('{}/index.yaml'.format(repository
.get('url')))
66 if r
.status_code
== 200:
67 repo_list
= yaml
.safe_load(r
.text
)
68 vnf_packages
= repo_list
.get('{}_packages'.format(pkgtype
))
69 for repo
in vnf_packages
:
70 versions
= vnf_packages
.get(repo
)
71 latest
= versions
.get('latest')
72 del versions
['latest']
73 for version
in versions
:
74 latest_version
= False
77 vnf_repos
.append({'vendor': versions
[version
].get("vendor"),
78 'name': versions
[version
].get("name"),
80 'description': versions
[version
].get("description"),
81 'location': versions
[version
].get("path"),
82 'repository': repository
.get('name'),
83 'repourl': repository
.get('url'),
84 'latest': latest_version
87 raise Exception('repository in url {} unreachable'.format(repository
.get('url')))
88 except Exception as e
:
89 logging
.error("Error cannot read from repository {} '{}': {}".format(repository
['name'], repository
['url'], e
))
92 vnf_repos_filtered
= []
94 for vnf_repo
in vnf_repos
:
95 for k
, v
in vnf_repo
.items():
97 kf
, vf
= filter.split('=')
98 if k
== kf
and vf
in v
:
99 vnf_repos_filtered
.append(vnf_repo
)
101 vnf_repos
= vnf_repos_filtered
104 def get_pkg(self
, pkgtype
, name
, repo
, filter, version
):
106 Returns the filename of the PKG downloaded to disk
108 self
._logger
.debug("")
109 self
._client
.get_token()
112 # Get OSM registered repository list
113 pkgs
= self
.pkg_list(pkgtype
, filter, repo
)
115 if pkg
.get('repository') == repo
and pkg
.get('name') == name
:
116 if 'latest' in version
:
117 if not pkg
.get('latest'):
120 version
= pkg
.get('version')
121 if pkg
.get('version') == version
:
122 r
= requests
.get('{}{}'.format(pkg
.get('repourl'), pkg
.get('location')), stream
=True)
123 if r
.status_code
!= 200:
124 raise ClientException("Package not found")
126 with tempfile
.NamedTemporaryFile(delete
=False) as f
:
127 f
.write(r
.raw
.read())
130 raise ClientException("{} {} not found at repo {}".format(pkgtype
,name
, repo
))
133 def pkg_get(self
, pkgtype
, name
, repo
, version
, filter):
135 pkg_name
= self
.get_pkg(pkgtype
, name
, repo
, filter, version
)
137 raise ClientException('Package not found')
138 folder
, descriptor
= self
.zip_extraction(pkg_name
)
139 with
open(descriptor
) as pkg
:
140 pkg_descriptor
= yaml
.safe_load(pkg
)
141 rmtree(folder
, ignore_errors
=False)
142 if ((pkgtype
== 'vnf' and (pkg_descriptor
.get('vnfd') or pkg_descriptor
.get('vnfd:vnfd_catalog'))) or
143 (pkgtype
== 'ns' and (pkg_descriptor
.get('nsd') or pkg_descriptor
.get('nsd:nsd_catalog')))):
144 raise ClientException('Wrong Package type')
145 return pkg_descriptor
147 def repo_index(self
, origin
=".", destination
='.'):
149 Repo Index main function
150 :param origin: origin directory for getting all the artifacts
151 :param destination: destination folder for create and index the valid artifacts
153 if destination
== '.':
154 if origin
== destination
:
155 destination
= 'repository'
157 destination
= abspath(destination
)
158 origin
= abspath(origin
)
161 origin
= join(getcwd(), origin
)
162 if destination
[0] != '/':
163 destination
= join(getcwd(), destination
)
165 self
.init_directory(destination
)
166 artifacts
= [f
for f
in listdir(origin
) if isfile(join(origin
, f
))]
167 directories
= [f
for f
in listdir(origin
) if isdir(join(origin
, f
))]
168 for artifact
in artifacts
:
169 self
.register_artifact_in_repository(join(origin
, artifact
), destination
, source
='file')
170 for artifact
in directories
:
171 self
.register_artifact_in_repository(join(origin
, artifact
), destination
, source
='directory')
172 print("\nFinal Results: ")
173 print("VNF Packages Indexed: " + str(len(glob
.glob(destination
+ "/vnf/*/*/metadata.yaml"))))
174 print("NS Packages Indexed: " + str(len(glob
.glob(destination
+ "/ns/*/*/metadata.yaml"))))
176 def md5(self
, fname
):
179 :param fname: file path
180 :return: checksum string
182 hash_md5
= hashlib
.md5()
183 with
open(fname
, "rb") as f
:
184 for chunk
in iter(lambda: f
.read(4096), b
""):
185 hash_md5
.update(chunk
)
186 return hash_md5
.hexdigest()
188 def fields_building(self
, descriptor_json
, file, package_type
):
190 From an artifact descriptor, obtain the fields required for indexing
191 :param descriptor_json: artifact description
192 :param file: artifact package
193 :param package_type: type of artifact (vnf or ns)
197 base_path
= '/{}/'.format(package_type
)
199 if package_type
== "vnf":
200 if descriptor_json
.get('vnfd-catalog', False):
201 aux_dict
= descriptor_json
.get('vnfd-catalog', {}).get('vnfd', [{}])[0]
203 aux_dict
= descriptor_json
.get('vnfd:vnfd-catalog', {}).get('vnfd', [{}])[0]
206 for vdu
in aux_dict
.get('vdu', ()):
207 images
.append(vdu
.get('image'))
208 fields
['images'] = images
209 if package_type
== "ns":
210 if descriptor_json
.get('nsd-catalog', False):
211 aux_dict
= descriptor_json
.get('nsd-catalog', {}).get('nsd', [{}])[0]
213 aux_dict
= descriptor_json
.get('nsd:nsd-catalog', {}).get('nsd', [{}])[0]
217 for vnf
in aux_dict
.get('constituent-vnfd', ()):
218 vnfs
.append(vnf
.get('vnfd-id-ref'))
219 self
._logger
.debug('Used VNFS in the NSD: ' + str(vnfs
))
220 fields
['vnfd-id-ref'] = vnfs
222 fields
['name'] = aux_dict
.get('name')
223 fields
['id'] = aux_dict
.get('id')
224 fields
['description'] = aux_dict
.get('description')
225 fields
['vendor'] = aux_dict
.get('vendor')
226 fields
['version'] = aux_dict
.get('version', '1.0')
227 fields
['path'] = "{}{}/{}/{}-{}.tar.gz".format(base_path
, fields
['id'], fields
['version'], fields
.get('id'), \
228 fields
.get('version'))
231 def zip_extraction(self
, file_name
):
233 Validation of artifact.
234 :param file: file path
235 :return: status details, status, fields, package_type
237 self
._logger
.debug("Decompressing package file")
238 temp_file
= '/tmp/{}'.format(file_name
.split('/')[-1])
239 if file_name
!= temp_file
:
240 copyfile(file_name
, temp_file
)
241 with tarfile
.open(temp_file
, "r:gz") as tar
:
242 folder
= tar
.getnames()[0].split('/')[0]
246 descriptor_file
= glob
.glob('{}/*.y*ml'.format(folder
))[0]
247 return folder
, descriptor_file
249 def validate_artifact(self
, path
, source
):
251 Validation of artifact.
252 :param path: file path
253 :return: status details, status, fields, package_type
258 if source
== 'directory':
259 descriptor_file
= glob
.glob('{}/*.y*ml'.format(path
))[0]
261 folder
, descriptor_file
= self
.zip_extraction(path
)
263 self
._logger
.debug("Opening descriptor file: {}".format(descriptor_file
))
265 with
open(descriptor_file
, 'r') as f
:
266 descriptor_data
= f
.read()
267 validation
= validation_im()
268 desc_type
, descriptor_data
= validation
.yaml_validation(descriptor_data
)
269 validation_im
.pyangbind_validation(self
, desc_type
, descriptor_data
)
270 if 'vnf' in list(descriptor_data
.keys())[0]:
273 # raise ClientException("Not VNF package")
276 self
._logger
.debug("Descriptor: {}".format(descriptor_data
))
277 fields
= self
.fields_building(descriptor_data
, path
, package_type
)
278 self
._logger
.debug("Descriptor sucessfully validated")
279 return {"detail": "{}D successfully validated".format(package_type
.upper()),
280 "code": "OK"}, True, fields
, package_type
281 except Exception as e
:
282 # Delete the folder we just created
283 return {"detail": str(e
)}, False, {}, package_type
286 rmtree(folder
, ignore_errors
=True)
288 def register_artifact_in_repository(self
, path
, destination
, source
):
290 Registration of one artifact in a repository
292 destination: path for index creation
298 _
, valid
, fields
, package_type
= self
.validate_artifact(path
, source
)
300 raise Exception('{} {} Not well configured.'.format(package_type
.upper(), str(path
)))
302 if source
== 'directory':
303 path
= pt
.build(path
)
305 fields
['checksum'] = self
.md5(path
)
306 self
.indexation(destination
, path
, package_type
, fields
)
308 except Exception as e
:
309 self
._logger
.debug("Error registering artifact in Repository: {}".format(e
))
312 if source
== 'directory' and compresed
:
315 def indexation(self
, destination
, path
, package_type
, fields
):
317 Process for index packages
318 :param destination: index repository path
319 :param path: path of the package
320 :param package_type: package type (vnf, ns)
321 :param fields: dict with the required values
323 data_ind
= {'name': fields
.get('name'), 'description': fields
.get('description'),
324 'vendor': fields
.get('vendor'), 'path': fields
.get('path')}
326 final_path
= join(destination
, package_type
, fields
.get('id'), fields
.get('version'))
327 if isdir(join(destination
, package_type
, fields
.get('id'))):
328 if isdir(final_path
):
329 self
._logger
.warning('{} {} already exists'.format(package_type
.upper(), str(path
)))
333 final_path
+ '/' + fields
.get('id') + "-" + fields
.get('version') + '.tar.gz')
334 yaml
.dump(fields
, open(final_path
+ '/' + 'metadata.yaml', 'w'),
335 Dumper
=ruamel
.yaml
.RoundTripDumper
)
336 index
= yaml
.load(open(destination
+ '/index.yaml'))
338 index
['{}_packages'.format(package_type
)][fields
.get('id')][fields
.get('version')] = data_ind
339 if versioning
.parse(index
['{}_packages'.format(package_type
)][fields
.get('id')][
340 'latest']) < versioning
.parse(fields
.get('version')):
341 index
['{}_packages'.format(package_type
)][fields
.get('id')]['latest'] = fields
.get(
343 yaml
.dump(index
, open(destination
+ '/index.yaml', 'w'), Dumper
=ruamel
.yaml
.RoundTripDumper
)
344 self
._logger
.info('{} {} added in the repository'.format(package_type
.upper(), str(path
)))
346 mkdir(destination
+ '/{}/'.format(package_type
) + fields
.get('id'))
349 final_path
+ '/' + fields
.get('id') + "-" + fields
.get('version') + '.tar.gz')
350 yaml
.dump(fields
, open(join(final_path
, 'metadata.yaml'), 'w'), Dumper
=ruamel
.yaml
.RoundTripDumper
)
351 index
= yaml
.load(open(destination
+ '/index.yaml'))
353 index
['{}_packages'.format(package_type
)][fields
.get('id')] = {fields
.get('version'): data_ind
}
354 index
['{}_packages'.format(package_type
)][fields
.get('id')]['latest'] = fields
.get('version')
355 yaml
.dump(index
, open(join(destination
, 'index.yaml'), 'w'), Dumper
=ruamel
.yaml
.RoundTripDumper
)
356 self
._logger
.info('{} {} added in the repository'.format(package_type
.upper(), str(path
)))
358 def current_datatime(self
):
361 :return: Datetime as string with the following structure "2020-04-29T08:41:07.681653Z"
363 return time
.strftime('%Y-%m-%dT%H:%M:%S.%sZ')
365 def init_directory(self
, destination
):
367 Initialize the index directory. Creation of index.yaml, and the directories for vnf and ns
371 if not isdir(destination
):
373 if not isfile(join(destination
, 'index.yaml')):
374 mkdir(join(destination
, 'vnf'))
375 mkdir(join(destination
, 'ns'))
376 index_data
= {'apiVersion': 'v1', 'generated': self
.current_datatime(), 'vnf_packages': {},
378 with
open(join(destination
, 'index.yaml'), 'w') as outfile
:
379 yaml
.dump(index_data
, outfile
, default_flow_style
=False)