a3b18402e85ef2e003d9d6a8d147e0ff3277f300
3 # Copyright 2016 RIFT.IO Inc
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
24 from . import checksums
29 class ArchiveError(Exception):
33 class ExtractError(Exception):
37 class PackageError(Exception):
41 class PackageValidationError(Exception):
45 class PackageAppendError(Exception):
49 class PackageFileChecksumError(PackageValidationError
):
50 def __init__(self
, filename
):
51 self
.filename
= filename
52 super().__init
__("Checksum mismatch for {}".format(filename
))
55 class DescriptorPackage(object):
56 """ This class provides an base class for a descriptor package representing
58 A descriptor package is a package which contains a single descriptor and any
59 associated files (logos, charms, scripts, etc). This package representation
60 attempts to be agnostic as to where the package files are being stored
61 (in memory, on disk, etc).
63 The package provides a simple interface to interact with the files within the
64 package and access the contained descriptor.
66 DESCRIPTOR_REGEX
= r
"{prefix}({descriptor_type}/[^/]*|[^/]*{descriptor_type})\.(xml|yml|yaml|json)$"
68 def __init__(self
, log
, open_fn
):
70 self
._open
_fn
= open_fn
72 self
._package
_file
_mode
_map
= {}
73 self
._package
_dirs
= set()
77 """ Return the leading parent directories shared by all files in the package
79 In order to remain flexible as to where tar was invoked to create the package,
80 the prefix represents the common parent directory path which all files in the
81 package have in common.
83 entries
= list(self
._package
_file
_mode
_map
) + list(self
._package
_dirs
)
86 prefix
= os
.path
.commonprefix(entries
)
87 if prefix
and not prefix
.endswith("/"):
89 elif len(entries
) == 1:
92 prefix
= os
.path
.dirname(entry
) + "/"
102 """ Return all files (with the prefix) in the package """
103 return list(self
._package
_file
_mode
_map
)
107 """ Return all directories in the package """
108 return list(self
._package
_dirs
)
111 def descriptor_type(self
):
112 """ A shorthand name for the type of descriptor (e.g. nsd)"""
113 raise NotImplementedError("Subclass must implement this property")
116 def serializer(self
):
117 """ An instance of convert.ProtoMessageSerializer """
118 raise NotImplementedError("Subclass must implement this property")
121 def descriptor_file(self
):
122 """ The descriptor file name (with prefix) """
123 regex
= self
.__class
__.DESCRIPTOR_REGEX
.format(
124 descriptor_type
=self
.descriptor_type
,
128 for filename
in self
.files
:
129 if re
.match(regex
, filename
):
130 if desc_file
is not None:
131 raise PackageError("Package contains more than one descriptor")
134 if desc_file
is None:
135 raise PackageError("Could not find descriptor file in package")
140 def descriptor_msg(self
):
141 """ The proto-GI descriptor message """
142 filename
= self
.descriptor_file
143 with self
.open(filename
) as hdl
:
144 _
, ext
= os
.path
.splitext(filename
)
145 nsd
= self
.serializer
.from_file_hdl(hdl
, ext
)
149 def json_descriptor(self
):
150 """ The JSON serialized descriptor message"""
151 nsd
= self
.descriptor_msg
152 return self
.serializer
.to_json_string(nsd
)
155 def descriptor_id(self
):
156 """ The descriptor id which uniquely identifies this descriptor in the system """
157 if not self
.descriptor_msg
.has_field("id"):
158 msg
= "Descriptor must have an id field"
160 raise PackageError(msg
)
162 return self
.descriptor_msg
.id
165 def descriptor_name(self
):
166 """ The descriptor name of this descriptor in the system """
167 if not self
.descriptor_msg
.has_field("name"):
168 msg
= "Descriptor name not present"
170 raise PackageError(msg
)
172 return self
.descriptor_msg
.name
175 def get_descriptor_patterns(cls
):
176 """ Returns a tuple of descriptor regex and Package Types """
177 package_types
= (VnfdPackage
, NsdPackage
)
180 for pkg_cls
in package_types
:
181 regex
= cls
.DESCRIPTOR_REGEX
.format(
182 descriptor_type
=pkg_cls
.DESCRIPTOR_TYPE
,
186 patterns
.append((regex
, pkg_cls
))
191 def from_package_files(cls
, log
, open_fn
, files
):
192 """ Creates a new DescriptorPackage subclass instance from a list of files
194 This classmethod detects the Package type from the package contents
195 and returns a new Package instance.
197 This will NOT subsequently add the files to the package so that must
198 be done by the client
202 open_fn - A function which can take a file name and mode and return
204 files - A list of files which would be added to the package after
208 A new DescriptorPackage subclass of the correct type for the descriptor
211 PackageError - Package type could not be determined from the list of files.
213 patterns
= cls
.get_descriptor_patterns()
217 for regex
, cls
in patterns
:
219 if re
.match(regex
, name
) is not None:
224 log
.error("No file in archive matched known descriptor formats: %s", regexes
)
225 raise PackageError("Could not determine package type from contents")
227 package
= pkg_cls(log
, open_fn
)
231 def from_descriptor_file_hdl(cls
, log
, file_hdl
):
232 """ Creates a new DescriptorPackage from a descriptor file handle
234 The descriptor file is added to the package before returning.
238 file_hdl - A file handle whose name attribute can be recognized as
239 particular descriptor type.
242 A new DescriptorPackage subclass of the correct type for the descriptor
245 PackageError - Package type could not be determined from the list of files.
246 ValueError - file_hdl did not have a name attribute provided
249 package_types
= (VnfdPackage
, NsdPackage
)
250 filename_patterns
= []
251 for package_cls
in package_types
:
252 filename_patterns
.append(
253 (r
".*{}.*".format(package_cls
.DESCRIPTOR_TYPE
), package_cls
)
256 if not hasattr(file_hdl
, 'name'):
257 raise ValueError("File descriptor must have a name attribute to create a descriptor package")
259 # Iterate through the recognized patterns and assign files accordingly
261 for pattern
, cls
in filename_patterns
:
262 if re
.match(pattern
, file_hdl
.name
):
267 raise PackageError("Could not determine package type from file name: %s" % file_hdl
.name
)
269 _
, ext
= os
.path
.splitext(file_hdl
.name
)
271 package_cls
.SERIALIZER
.from_file_hdl(file_hdl
, ext
)
272 except convert
.SerializationError
as e
:
273 raise PackageError("Could not deserialize descriptor %s" % file_hdl
.name
) from e
275 # Create a new file handle for each open call to prevent independent clients
276 # from affecting each other
278 new_hdl
= io
.BytesIO(file_hdl
.read())
280 def do_open(file_path
):
281 assert file_path
== file_hdl
.name
282 hdl
= io
.BytesIO(new_hdl
.getvalue())
285 desc_pkg
= package_cls(log
, do_open
)
286 desc_pkg
.add_file(file_hdl
.name
)
290 def get_file_mode(self
, pkg_file
):
291 """ Returns the file mode for the package file
294 pkg_file - A file name in the package
300 PackageError - The file does not exist in the package
303 return self
._package
_file
_mode
_map
[pkg_file
]
304 except KeyError as e
:
305 msg
= "Could not find package_file: %s" % pkg_file
307 raise PackageError(msg
) from e
309 def extract_dir(self
, src_dir
, dest_root_dir
, extract_images
=False):
310 """ Extract a specific directory contents to dest_root_dir
313 src_dir - A directory within the package (None means all files/directories)
314 dest_root_dir - A directory to extract directory contents to
315 extract_images - A flag indicating whether we want to extract images
318 ExtractError - Directory contents could not be extracted
320 if src_dir
is not None and src_dir
not in self
._package
_dirs
:
321 raise ExtractError("Could not find source dir: %s" % src_dir
)
323 for filename
in self
.files
:
324 if not extract_images
and image
.is_image_file(filename
):
327 if src_dir
is not None and not filename
.startswith(src_dir
):
330 # Copy the contents of the file to the correct path
331 # Remove the common prefix and create the dest filename
332 if src_dir
is not None:
333 fname
= filename
[len(src_dir
):]
338 dest_file_path
= os
.path
.join(dest_root_dir
, fname
)
339 dest_dir_path
= os
.path
.dirname(dest_file_path
)
340 if not os
.path
.exists(dest_dir_path
):
341 os
.makedirs(dest_dir_path
)
343 with
open(dest_file_path
, 'wb') as dst_hdl
:
344 with self
.open(filename
) as src_hdl
:
345 shutil
.copyfileobj(src_hdl
, dst_hdl
, 10 * 1024 * 1024)
347 # Set the file mode to original
348 os
.chmod(dest_file_path
, self
._package
_file
_mode
_map
[filename
])
350 def insert_file(self
, new_file
, dest_file
, rel_path
, mode
=0o777):
351 self
.add_file(rel_path
, mode
)
354 # Copy the contents of the file to the correct path
355 dest_dir_path
= os
.path
.dirname(dest_file
)
356 if not os
.path
.isdir(dest_dir_path
):
357 os
.makedirs(dest_dir_path
)
359 with
open(dest_file
, 'wb') as dst_hdl
:
360 with
open(new_file
, 'rb') as src_hdl
:
361 shutil
.copyfileobj(src_hdl
, dst_hdl
, 10 * 1024 * 1024)
363 # Set the file mode to original
364 os
.chmod(dest_file
, self
._package
_file
_mode
_map
[rel_path
])
365 except Exception as e
:
366 # Clear the file when an exception happens
367 if os
.path
.isfile(dest_file
):
370 raise PackageAppendError(str(e
))
372 def delete_file(self
, dest_file
, rel_path
):
373 self
.remove_file(rel_path
)
377 except Exception as e
:
378 raise PackageAppendError(str(e
))
380 def extract_file(self
, src_file
, dest_file
):
381 """ Extract a specific package file to dest_file
383 The destination directory will be created if it does not exist.
386 src_file - A file within the package
387 dest_file - A file path to extract file contents to
390 ExtractError - Directory contents could not be extracted
392 if src_file
not in self
._package
_file
_mode
_map
:
393 msg
= "Could not find source file %s" % src_file
395 raise ExtractError(msg
)
397 # Copy the contents of the file to the correct path
398 dest_dir_path
= os
.path
.dirname(dest_file
)
399 if not os
.path
.isdir(dest_dir_path
):
400 os
.makedirs(dest_dir_path
)
402 with
open(dest_file
, 'wb') as dst_hdl
:
403 with self
.open(src_file
) as src_hdl
:
404 shutil
.copyfileobj(src_hdl
, dst_hdl
, 10 * 1024 * 1024)
406 # Set the file mode to original
407 os
.chmod(dest_file
, self
._package
_file
_mode
_map
[src_file
])
409 def extract(self
, dest_root_dir
, extract_images
=False):
410 """ Extract all package contents to a destination directory
413 dest_root_dir - The directory to extract package contents to
416 NotADirectoryError - dest_root_dir is not a directory
418 if not os
.path
.isdir(dest_root_dir
):
419 raise NotADirectoryError(dest_root_dir
)
421 self
.extract_dir(None, dest_root_dir
, extract_images
)
423 def open(self
, rel_path
):
424 """ Open a file contained in the package in read-only, binary mode.
427 rel_path - The file path within the package
430 A file-like object opened in read-only mode.
433 PackageError - The file could not be opened
436 return self
._open
_fn
(rel_path
)
437 except Exception as e
:
438 msg
= "Could not open file from package: %s" % rel_path
439 self
._log
.warning(msg
)
440 raise PackageError(msg
) from e
442 def add_file(self
, rel_path
, mode
=0o777):
443 """ Add a file to the package.
445 The file should be specified as a relative path to the package
446 root. The open_fn provided in the constructor must be able to
447 take the relative path and open the actual source file from
448 wherever the file actually is stored.
450 If the file's parent directories do not yet exist, add them to
454 rel_path - The file path relative to the top of the package.
455 mode - The permission mode the file should be stored with so
456 it can be extracted with the correct permissions.
459 PackageError - The file could not be added to the package
462 raise PackageError("Empty file name added")
464 if rel_path
in self
._package
_file
_mode
_map
:
465 raise PackageError("File %s already exists in package" % rel_path
)
467 # If the file's directory is not in the package add it.
468 rel_dir
= os
.path
.dirname(rel_path
)
470 self
._package
_dirs
.add(rel_dir
)
471 rel_dir
= os
.path
.dirname(rel_dir
)
473 self
._package
_file
_mode
_map
[rel_path
] = mode
475 def remove_file(self
, rel_path
):
477 raise PackageError("Empty file name added")
479 if rel_path
not in self
._package
_file
_mode
_map
:
480 raise PackageError("File %s does not in package" % rel_path
)
482 del self
._package
_file
_mode
_map
[rel_path
]
484 def add_dir(self
, rel_path
):
485 """ Add a directory to the package
488 rel_path - The directories relative path.
491 PackageError - A file already exists in the package with the same name.
493 if rel_path
in self
._package
_file
_mode
_map
:
494 raise PackageError("File already exists with the same name: %s", rel_path
)
496 if rel_path
in self
._package
_dirs
:
497 self
._log
.warning("%s directory already exists", rel_path
)
500 self
._package
_dirs
.add(rel_path
)
503 class NsdPackage(DescriptorPackage
):
504 DESCRIPTOR_TYPE
= "nsd"
505 SERIALIZER
= convert
.RwNsdSerializer()
508 def descriptor_type(self
):
512 def serializer(self
):
513 return NsdPackage
.SERIALIZER
516 class VnfdPackage(DescriptorPackage
):
517 DESCRIPTOR_TYPE
= "vnfd"
518 SERIALIZER
= convert
.RwVnfdSerializer()
521 def descriptor_type(self
):
525 def serializer(self
):
526 return VnfdPackage
.SERIALIZER
529 class PackageChecksumValidator(object):
530 """ This class uses the checksums.txt file in the package
531 and validates that all files in the package match the checksum that exists within
534 CHECKSUM_FILE
= "{prefix}checksums.txt"
536 def __init__(self
, log
):
540 def get_package_checksum_file(cls
, package
):
541 checksum_file
= cls
.CHECKSUM_FILE
.format(prefix
=package
.prefix
)
542 if checksum_file
not in package
.files
:
543 raise FileNotFoundError("%s does not exist in archive" % checksum_file
)
547 def validate(self
, package
):
548 """ Validate file checksums match that in the checksums.txt
551 package - The Descriptor Package which possiblity contains checksums.txt
553 Returns: A dictionary of files that were validated by the checksums.txt
554 along with their checksums
557 PackageValidationError - The package validation failed for some
559 PackageFileChecksumError - A file within the package did not match the
560 checksum within checksums.txt
562 validated_file_checksums
= {}
565 checksum_file
= PackageChecksumValidator
.get_package_checksum_file(package
)
566 with package
.open(checksum_file
) as checksum_hdl
:
567 archive_checksums
= checksums
.ArchiveChecksums
.from_file_desc(checksum_hdl
)
568 except (FileNotFoundError
, PackageError
) as e
:
569 self
._log
.warning("Could not open package checksum file. Not validating checksums.")
570 return validated_file_checksums
572 for pkg_file
in package
.files
:
573 if pkg_file
== checksum_file
:
576 pkg_file_no_prefix
= pkg_file
.replace(package
.prefix
, "", 1)
577 if pkg_file_no_prefix
not in archive_checksums
:
578 self
._log
.warning("File %s not found in checksum file %s",
579 pkg_file
, checksum_file
)
583 with package
.open(pkg_file
) as pkg_file_hdl
:
584 file_checksum
= checksums
.checksum(pkg_file_hdl
)
585 except PackageError
as e
:
586 msg
= "Could not read package file {} for checksum validation: {}".format(
589 raise PackageValidationError(msg
) from e
591 if archive_checksums
[pkg_file_no_prefix
] != file_checksum
:
592 msg
= "{} checksum ({}) did match expected checksum ({})".format(
593 pkg_file
, file_checksum
, archive_checksums
[pkg_file_no_prefix
]
596 raise PackageFileChecksumError(pkg_file
)
598 validated_file_checksums
[pkg_file
] = file_checksum
600 return validated_file_checksums
603 class TarPackageArchive(object):
604 """ This class represents a package stored within a tar.gz archive file """
605 def __init__(self
, log
, tar_file_hdl
, mode
="r"):
607 self
._tar
_filepath
= tar_file_hdl
610 self
._tarfile
= tarfile
.open(fileobj
=tar_file_hdl
, mode
=mode
)
615 return "TarPackageArchive(%s)" % self
._tar
_filepath
617 def _get_members(self
):
618 return [info
for info
in self
._tarfile
.getmembers()]
620 def _load_archive(self
):
621 self
._tar
_infos
= {info
.name
: info
for info
in self
._get
_members
() if info
.name
}
627 """ Close the opened tarfile"""
628 if self
._tarfile
is not None:
629 self
._tarfile
.close()
634 """ The list of file members within the tar file """
635 return [name
for name
in self
._tar
_infos
if tarfile
.TarInfo
.isfile(self
._tar
_infos
[name
])]
637 def open_file(self
, rel_file_path
):
638 """ Opens a file within the archive as read-only, byte mode.
641 rel_file_path - The file path within the archive to open
644 A file like object (see tarfile.extractfile())
647 ArchiveError - The file could not be opened for some generic reason.
649 if rel_file_path
not in self
._tar
_infos
:
650 raise ArchiveError("Could not find %s in tar file", rel_file_path
)
653 return self
._tarfile
.extractfile(rel_file_path
)
654 except tarfile
.TarError
as e
:
655 msg
= "Failed to read file {} from tarfile {}: {}".format(
656 rel_file_path
, self
._tar
_filepath
, str(e
)
659 raise ArchiveError(msg
) from e
661 def create_package(self
):
662 """ Creates a Descriptor package from the archive contents
665 A DescriptorPackage of the correct descriptor type
667 package
= DescriptorPackage
.from_package_files(self
._log
, self
.open_file
, self
.filenames
)
668 for pkg_file
in self
.filenames
:
669 package
.add_file(pkg_file
, self
._tar
_infos
[pkg_file
].mode
)
674 class TemporaryPackage(object):
675 """ This class is a container for a temporary file-backed package
677 This class contains a DescriptorPackage and can be used in place of one.
678 Provides a useful context manager which will close and destroy the file
679 that is backing the DescriptorPackage on exit.
681 def __init__(self
, log
, package
, file_hdl
):
683 self
._package
= package
684 self
._file
_hdl
= file_hdl
686 if not hasattr(self
._file
_hdl
, "name"):
687 raise ValueError("File handle must have a name attribute")
689 def __getattr__(self
, attr
):
690 return getattr(self
._package
, attr
)
695 def __exit__(self
, type, value
, tb
):
699 """ Returns the filepath with is backing the Package """
700 return self
._file
_hdl
.name
703 """ The contained DescriptorPackage instance """
707 """ Close and remove the backed file """
708 filename
= self
._file
_hdl
.name
711 self
._file
_hdl
.close()
713 self
._log
.warning("Failed to close package file: %s", str(e
))
718 self
._log
.warning("Failed to remove package file: %s", str(e
))