3 # Copyright 2016 RIFT.IO Inc
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
24 from . import checksums
29 class ArchiveError(Exception):
33 class ExtractError(Exception):
37 class PackageError(Exception):
41 class PackageValidationError(Exception):
45 class PackageAppendError(Exception):
49 class PackageFileChecksumError(PackageValidationError
):
50 def __init__(self
, filename
):
51 self
.filename
= filename
52 super().__init
__("Checksum mismatch for {}".format(filename
))
55 class DescriptorPackage(object):
56 """ This class provides an base class for a descriptor package representing
58 A descriptor package is a package which contains a single descriptor and any
59 associated files (logos, charms, scripts, etc). This package representation
60 attempts to be agnostic as to where the package files are being stored
61 (in memory, on disk, etc).
63 The package provides a simple interface to interact with the files within the
64 package and access the contained descriptor.
66 DESCRIPTOR_REGEX
= r
"{prefix}({descriptor_type}/[^/]*|[^/]*{descriptor_type})\.(xml|yml|yaml|json)$"
68 def __init__(self
, log
, open_fn
):
70 self
._open
_fn
= open_fn
72 self
._package
_file
_mode
_map
= {}
73 self
._package
_dirs
= set()
77 """ Return the leading parent directories shared by all files in the package
79 In order to remain flexible as to where tar was invoked to create the package,
80 the prefix represents the common parent directory path which all files in the
81 package have in common.
83 entries
= list(self
._package
_file
_mode
_map
) + list(self
._package
_dirs
)
86 prefix
= os
.path
.commonprefix(entries
)
87 if prefix
and not prefix
.endswith("/"):
89 elif len(entries
) == 1:
92 prefix
= os
.path
.dirname(entry
) + "/"
102 """ Return all files (with the prefix) in the package """
103 return list(self
._package
_file
_mode
_map
)
107 """ Return all directories in the package """
108 return list(self
._package
_dirs
)
111 def descriptor_type(self
):
112 """ A shorthand name for the type of descriptor (e.g. nsd)"""
113 raise NotImplementedError("Subclass must implement this property")
116 def serializer(self
):
117 """ An instance of convert.ProtoMessageSerializer """
118 raise NotImplementedError("Subclass must implement this property")
121 def descriptor_file(self
):
122 """ The descriptor file name (with prefix) """
123 regex
= self
.__class
__.DESCRIPTOR_REGEX
.format(
124 descriptor_type
=self
.descriptor_type
,
128 for filename
in self
.files
:
129 if re
.match(regex
, filename
):
130 if desc_file
is not None:
131 raise PackageError("Package contains more than one descriptor")
134 if desc_file
is None:
135 raise PackageError("Could not find descriptor file in package")
140 def descriptor_msg(self
):
141 """ The proto-GI descriptor message """
142 filename
= self
.descriptor_file
143 with self
.open(filename
) as hdl
:
144 _
, ext
= os
.path
.splitext(filename
)
145 nsd
= self
.serializer
.from_file_hdl(hdl
, ext
)
149 def json_descriptor(self
):
150 """ The JSON serialized descriptor message"""
151 nsd
= self
.descriptor_msg
152 return self
.serializer
.to_json_string(nsd
)
155 def descriptor_id(self
):
156 """ The descriptor id which uniquely identifies this descriptor in the system """
157 if not self
.descriptor_msg
.has_field("id"):
158 msg
= "Descriptor must have an id field"
160 raise PackageError(msg
)
162 return self
.descriptor_msg
.id
165 def descriptor_name(self
):
166 """ The descriptor name of this descriptor in the system """
167 if not self
.descriptor_msg
.has_field("name"):
168 msg
= "Descriptor name not present"
170 raise PackageError(msg
)
172 return self
.descriptor_msg
.name
175 def descriptor_version(self
):
176 desc_msg
= self
.descriptor_msg
177 return desc_msg
.version
if desc_msg
.has_field("version") else ''
180 def descriptor_vendor(self
):
181 desc_msg
= self
.descriptor_msg
182 return desc_msg
.vendor
if desc_msg
.has_field("vendor") else ''
185 def get_descriptor_patterns(cls
):
186 """ Returns a tuple of descriptor regex and Package Types """
187 package_types
= (VnfdPackage
, NsdPackage
)
190 for pkg_cls
in package_types
:
191 regex
= cls
.DESCRIPTOR_REGEX
.format(
192 descriptor_type
=pkg_cls
.DESCRIPTOR_TYPE
,
196 patterns
.append((regex
, pkg_cls
))
201 def from_package_files(cls
, log
, open_fn
, files
):
202 """ Creates a new DescriptorPackage subclass instance from a list of files
204 This classmethod detects the Package type from the package contents
205 and returns a new Package instance.
207 This will NOT subsequently add the files to the package so that must
208 be done by the client
212 open_fn - A function which can take a file name and mode and return
214 files - A list of files which would be added to the package after
218 A new DescriptorPackage subclass of the correct type for the descriptor
221 PackageError - Package type could not be determined from the list of files.
223 patterns
= cls
.get_descriptor_patterns()
227 for regex
, cls
in patterns
:
229 if re
.match(regex
, name
) is not None:
234 log
.error("No file in archive matched known descriptor formats: %s", regexes
)
235 raise PackageError("Could not determine package type from contents")
237 package
= pkg_cls(log
, open_fn
)
241 def from_descriptor_file_hdl(cls
, log
, file_hdl
):
242 """ Creates a new DescriptorPackage from a descriptor file handle
244 The descriptor file is added to the package before returning.
248 file_hdl - A file handle whose name attribute can be recognized as
249 particular descriptor type.
252 A new DescriptorPackage subclass of the correct type for the descriptor
255 PackageError - Package type could not be determined from the list of files.
256 ValueError - file_hdl did not have a name attribute provided
259 package_types
= (VnfdPackage
, NsdPackage
)
260 filename_patterns
= []
261 for package_cls
in package_types
:
262 filename_patterns
.append(
263 (r
".*{}.*".format(package_cls
.DESCRIPTOR_TYPE
), package_cls
)
266 if not hasattr(file_hdl
, 'name'):
267 raise ValueError("File descriptor must have a name attribute to create a descriptor package")
269 # Iterate through the recognized patterns and assign files accordingly
271 for pattern
, cls
in filename_patterns
:
272 if re
.match(pattern
, file_hdl
.name
):
277 raise PackageError("Could not determine package type from file name: %s" % file_hdl
.name
)
279 _
, ext
= os
.path
.splitext(file_hdl
.name
)
281 package_cls
.SERIALIZER
.from_file_hdl(file_hdl
, ext
)
282 except convert
.SerializationError
as e
:
283 raise PackageError("Could not deserialize descriptor %s" % file_hdl
.name
) from e
285 # Create a new file handle for each open call to prevent independent clients
286 # from affecting each other
288 new_hdl
= io
.BytesIO(file_hdl
.read())
290 def do_open(file_path
):
291 assert file_path
== file_hdl
.name
292 hdl
= io
.BytesIO(new_hdl
.getvalue())
295 desc_pkg
= package_cls(log
, do_open
)
296 desc_pkg
.add_file(file_hdl
.name
)
300 def get_file_mode(self
, pkg_file
):
301 """ Returns the file mode for the package file
304 pkg_file - A file name in the package
310 PackageError - The file does not exist in the package
313 return self
._package
_file
_mode
_map
[pkg_file
]
314 except KeyError as e
:
315 msg
= "Could not find package_file: %s" % pkg_file
317 raise PackageError(msg
) from e
319 def extract_dir(self
, src_dir
, dest_root_dir
, extract_images
=False):
320 """ Extract a specific directory contents to dest_root_dir
323 src_dir - A directory within the package (None means all files/directories)
324 dest_root_dir - A directory to extract directory contents to
325 extract_images - A flag indicating whether we want to extract images
328 ExtractError - Directory contents could not be extracted
330 if src_dir
is not None and src_dir
not in self
._package
_dirs
:
331 raise ExtractError("Could not find source dir: %s" % src_dir
)
333 for filename
in self
.files
:
334 if not extract_images
and image
.is_image_file(filename
):
337 if src_dir
is not None and not filename
.startswith(src_dir
):
340 # Copy the contents of the file to the correct path
341 # Remove the common prefix and create the dest filename
342 if src_dir
is not None:
343 fname
= filename
[len(src_dir
):]
348 dest_file_path
= os
.path
.join(dest_root_dir
, fname
)
349 dest_dir_path
= os
.path
.dirname(dest_file_path
)
350 if not os
.path
.exists(dest_dir_path
):
351 os
.makedirs(dest_dir_path
)
353 with
open(dest_file_path
, 'wb') as dst_hdl
:
354 with self
.open(filename
) as src_hdl
:
355 shutil
.copyfileobj(src_hdl
, dst_hdl
, 10 * 1024 * 1024)
357 # Set the file mode to original
358 os
.chmod(dest_file_path
, self
._package
_file
_mode
_map
[filename
])
360 def insert_file(self
, new_file
, dest_file
, rel_path
, mode
=0o777):
361 self
.add_file(rel_path
, mode
)
364 # Copy the contents of the file to the correct path
365 # For folder creation (or nested folders), dest_file appears w/ trailing "/" like: dir1/ or dir1/dir2/
366 # For regular file upload, dest_file appears as dir1/abc.txt
368 dest_dir_path
= os
.path
.dirname(dest_file
)
369 if not os
.path
.isdir(dest_dir_path
):
370 os
.makedirs(dest_dir_path
)
371 if not os
.path
.basename(dest_file
):
372 self
._log
.debug("Created dir path, no filename to insert in {}, skipping..".format(dest_dir_path
))
375 with
open(dest_file
, 'wb') as dst_hdl
:
376 with
open(new_file
, 'rb') as src_hdl
:
377 shutil
.copyfileobj(src_hdl
, dst_hdl
, 10 * 1024 * 1024)
379 # Set the file mode to original
380 os
.chmod(dest_file
, self
._package
_file
_mode
_map
[rel_path
])
381 except Exception as e
:
382 # Clear the file when an exception happens
383 if os
.path
.isfile(dest_file
):
386 raise PackageAppendError(str(e
))
388 def delete_file(self
, dest_file
, rel_path
):
389 self
.remove_file(rel_path
)
393 except Exception as e
:
394 raise PackageAppendError(str(e
))
396 def extract_file(self
, src_file
, dest_file
):
397 """ Extract a specific package file to dest_file
399 The destination directory will be created if it does not exist.
402 src_file - A file within the package
403 dest_file - A file path to extract file contents to
406 ExtractError - Directory contents could not be extracted
408 if src_file
not in self
._package
_file
_mode
_map
:
409 msg
= "Could not find source file %s" % src_file
411 raise ExtractError(msg
)
413 # Copy the contents of the file to the correct path
414 dest_dir_path
= os
.path
.dirname(dest_file
)
415 if not os
.path
.isdir(dest_dir_path
):
416 os
.makedirs(dest_dir_path
)
418 with
open(dest_file
, 'wb') as dst_hdl
:
419 with self
.open(src_file
) as src_hdl
:
420 shutil
.copyfileobj(src_hdl
, dst_hdl
, 10 * 1024 * 1024)
422 # Set the file mode to original
423 os
.chmod(dest_file
, self
._package
_file
_mode
_map
[src_file
])
425 def extract(self
, dest_root_dir
, extract_images
=False):
426 """ Extract all package contents to a destination directory
429 dest_root_dir - The directory to extract package contents to
432 NotADirectoryError - dest_root_dir is not a directory
434 if not os
.path
.isdir(dest_root_dir
):
435 raise NotADirectoryError(dest_root_dir
)
437 self
.extract_dir(None, dest_root_dir
, extract_images
)
439 def open(self
, rel_path
):
440 """ Open a file contained in the package in read-only, binary mode.
443 rel_path - The file path within the package
446 A file-like object opened in read-only mode.
449 PackageError - The file could not be opened
452 return self
._open
_fn
(rel_path
)
453 except Exception as e
:
454 msg
= "Could not open file from package: %s" % rel_path
455 self
._log
.warning(msg
)
456 raise PackageError(msg
) from e
458 def add_file(self
, rel_path
, mode
=0o777):
459 """ Add a file to the package.
461 The file should be specified as a relative path to the package
462 root. The open_fn provided in the constructor must be able to
463 take the relative path and open the actual source file from
464 wherever the file actually is stored.
466 If the file's parent directories do not yet exist, add them to
470 rel_path - The file path relative to the top of the package.
471 mode - The permission mode the file should be stored with so
472 it can be extracted with the correct permissions.
475 PackageError - The file could not be added to the package
478 raise PackageError("Empty file name added")
480 if rel_path
in self
._package
_file
_mode
_map
:
481 raise PackageError("File %s already exists in package" % rel_path
)
483 # If the file's directory is not in the package add it.
484 rel_dir
= os
.path
.dirname(rel_path
)
486 self
._package
_dirs
.add(rel_dir
)
487 rel_dir
= os
.path
.dirname(rel_dir
)
489 self
._package
_file
_mode
_map
[rel_path
] = mode
491 def remove_file(self
, rel_path
):
493 raise PackageError("Empty file name added")
495 if rel_path
not in self
._package
_file
_mode
_map
:
496 raise PackageError("File %s does not exist in package" % rel_path
)
498 del self
._package
_file
_mode
_map
[rel_path
]
500 def add_dir(self
, rel_path
):
501 """ Add a directory to the package
504 rel_path - The directories relative path.
507 PackageError - A file already exists in the package with the same name.
509 if rel_path
in self
._package
_file
_mode
_map
:
510 raise PackageError("File already exists with the same name: %s", rel_path
)
512 if rel_path
in self
._package
_dirs
:
513 self
._log
.warning("%s directory already exists", rel_path
)
516 self
._package
_dirs
.add(rel_path
)
519 class NsdPackage(DescriptorPackage
):
520 DESCRIPTOR_TYPE
= "nsd"
521 SERIALIZER
= convert
.RwNsdSerializer()
524 def descriptor_type(self
):
528 def serializer(self
):
529 return NsdPackage
.SERIALIZER
532 class VnfdPackage(DescriptorPackage
):
533 DESCRIPTOR_TYPE
= "vnfd"
534 SERIALIZER
= convert
.RwVnfdSerializer()
537 def descriptor_type(self
):
541 def serializer(self
):
542 return VnfdPackage
.SERIALIZER
544 class PackageConstructValidator(object):
546 def __init__(self
, log
):
549 def validate(self
, package
):
550 """ Validate presence of descriptor file (.yaml) at the top level in the
551 package folder structure.
554 package - The Descriptor Package being validated.
558 PackageValidationError - The package validation failed for some
562 desc_file
= package
.descriptor_file
563 prefix
, desc_file
= package
.prefix
.rstrip('/'), desc_file
.rstrip('/')
565 if os
.path
.dirname(desc_file
) != prefix
:
566 msg
= "Descriptor file {} not found in expcted location {}".format(desc_file
, prefix
)
568 raise PackageValidationError(msg
)
571 class PackageChecksumValidator(object):
572 """ This class uses the checksums.txt file in the package
573 and validates that all files in the package match the checksum that exists within
576 CHECKSUM_FILE
= "{prefix}checksums.txt"
578 def __init__(self
, log
):
580 self
.validated_file_checksums
= {}
583 def get_package_checksum_file(cls
, package
):
584 checksum_file
= cls
.CHECKSUM_FILE
.format(prefix
=package
.prefix
)
585 if checksum_file
not in package
.files
:
586 raise FileNotFoundError("%s does not exist in archive" % checksum_file
)
592 return self
.validated_file_checksums
594 def validate(self
, package
):
595 """ Validate file checksums match that in the checksums.txt
598 package - The Descriptor Package which possiblity contains checksums.txt
600 Returns: A dictionary of files that were validated by the checksums.txt
601 along with their checksums
604 PackageValidationError - The package validation failed for some
606 PackageFileChecksumError - A file within the package did not match the
607 checksum within checksums.txt
611 checksum_file
= PackageChecksumValidator
.get_package_checksum_file(package
)
612 with package
.open(checksum_file
) as checksum_hdl
:
613 archive_checksums
= checksums
.ArchiveChecksums
.from_file_desc(checksum_hdl
)
614 except (FileNotFoundError
, PackageError
) as e
:
615 self
._log
.warning("Could not open package checksum file. Not validating checksums.")
616 return validated_file_checksums
618 for pkg_file
in package
.files
:
619 if pkg_file
== checksum_file
:
622 pkg_file_no_prefix
= pkg_file
.replace(package
.prefix
, "", 1)
623 if pkg_file_no_prefix
not in archive_checksums
:
624 self
._log
.warning("File %s not found in checksum file %s",
625 pkg_file
, checksum_file
)
629 with package
.open(pkg_file
) as pkg_file_hdl
:
630 file_checksum
= checksums
.checksum(pkg_file_hdl
)
631 except PackageError
as e
:
632 msg
= "Could not read package file {} for checksum validation: {}".format(
635 raise PackageValidationError(msg
) from e
637 if archive_checksums
[pkg_file_no_prefix
] != file_checksum
:
638 msg
= "{} checksum ({}) did match expected checksum ({})".format(
639 pkg_file
, file_checksum
, archive_checksums
[pkg_file_no_prefix
]
642 raise PackageFileChecksumError(pkg_file
)
644 self
.validated_file_checksums
[pkg_file
] = file_checksum
647 class TarPackageArchive(object):
648 """ This class represents a package stored within a tar.gz archive file """
649 def __init__(self
, log
, tar_file_hdl
, mode
="r"):
651 self
._tar
_filepath
= tar_file_hdl
654 self
._tarfile
= tarfile
.open(fileobj
=tar_file_hdl
, mode
=mode
)
659 return "TarPackageArchive(%s)" % self
._tar
_filepath
661 def _get_members(self
):
662 return [info
for info
in self
._tarfile
.getmembers()]
664 def _load_archive(self
):
665 self
._tar
_infos
= {info
.name
: info
for info
in self
._get
_members
() if info
.name
}
671 """ Close the opened tarfile"""
672 if self
._tarfile
is not None:
673 self
._tarfile
.close()
678 """ The list of file members within the tar file """
679 return [name
for name
in self
._tar
_infos
if tarfile
.TarInfo
.isfile(self
._tar
_infos
[name
])]
681 def open_file(self
, rel_file_path
):
682 """ Opens a file within the archive as read-only, byte mode.
685 rel_file_path - The file path within the archive to open
688 A file like object (see tarfile.extractfile())
691 ArchiveError - The file could not be opened for some generic reason.
693 if rel_file_path
not in self
._tar
_infos
:
694 raise ArchiveError("Could not find %s in tar file", rel_file_path
)
697 return self
._tarfile
.extractfile(rel_file_path
)
698 except tarfile
.TarError
as e
:
699 msg
= "Failed to read file {} from tarfile {}: {}".format(
700 rel_file_path
, self
._tar
_filepath
, str(e
)
703 raise ArchiveError(msg
) from e
705 def create_package(self
):
706 """ Creates a Descriptor package from the archive contents
709 A DescriptorPackage of the correct descriptor type
711 package
= DescriptorPackage
.from_package_files(self
._log
, self
.open_file
, self
.filenames
)
712 for pkg_file
in self
.filenames
:
713 package
.add_file(pkg_file
, self
._tar
_infos
[pkg_file
].mode
)
718 class TemporaryPackage(object):
719 """ This class is a container for a temporary file-backed package
721 This class contains a DescriptorPackage and can be used in place of one.
722 Provides a useful context manager which will close and destroy the file
723 that is backing the DescriptorPackage on exit.
725 def __init__(self
, log
, package
, file_hdl
):
727 self
._package
= package
728 self
._file
_hdl
= file_hdl
730 if not hasattr(self
._file
_hdl
, "name"):
731 raise ValueError("File handle must have a name attribute")
733 def __getattr__(self
, attr
):
734 return getattr(self
._package
, attr
)
739 def __exit__(self
, type, value
, tb
):
743 """ Returns the filepath with is backing the Package """
744 return self
._file
_hdl
.name
747 """ The contained DescriptorPackage instance """
751 """ Close and remove the backed file """
752 filename
= self
._file
_hdl
.name
755 self
._file
_hdl
.close()
757 self
._log
.warning("Failed to close package file: %s", str(e
))
762 self
._log
.warning("Failed to remove package file: %s", str(e
))