3 # Copyright 2016 RIFT.IO Inc
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
24 from . import checksums
29 class ArchiveError(Exception):
33 class ExtractError(Exception):
37 class PackageError(Exception):
41 class PackageValidationError(Exception):
45 class PackageAppendError(Exception):
49 class PackageFileChecksumError(PackageValidationError
):
50 def __init__(self
, filename
):
51 self
.filename
= filename
52 super().__init
__("Checksum mismatch for {}".format(filename
))
55 class DescriptorPackage(object):
56 """ This class provides an base class for a descriptor package representing
58 A descriptor package is a package which contains a single descriptor and any
59 associated files (logos, charms, scripts, etc). This package representation
60 attempts to be agnostic as to where the package files are being stored
61 (in memory, on disk, etc).
63 The package provides a simple interface to interact with the files within the
64 package and access the contained descriptor.
66 DESCRIPTOR_REGEX
= r
"{prefix}({descriptor_type}/[^/]*|[^/]*{descriptor_type})\.(xml|yml|yaml|json)$"
68 def __init__(self
, log
, open_fn
):
70 self
._open
_fn
= open_fn
72 self
._package
_file
_mode
_map
= {}
73 self
._package
_dirs
= set()
77 """ Return the leading parent directories shared by all files in the package
79 In order to remain flexible as to where tar was invoked to create the package,
80 the prefix represents the common parent directory path which all files in the
81 package have in common.
83 entries
= list(self
._package
_file
_mode
_map
) + list(self
._package
_dirs
)
86 prefix
= os
.path
.commonprefix(entries
)
87 if prefix
and not prefix
.endswith("/"):
89 elif len(entries
) == 1:
92 prefix
= os
.path
.dirname(entry
) + "/"
102 """ Return all files (with the prefix) in the package """
103 return list(self
._package
_file
_mode
_map
)
107 """ Return all directories in the package """
108 return list(self
._package
_dirs
)
111 def descriptor_type(self
):
112 """ A shorthand name for the type of descriptor (e.g. nsd)"""
113 raise NotImplementedError("Subclass must implement this property")
116 def serializer(self
):
117 """ An instance of convert.ProtoMessageSerializer """
118 raise NotImplementedError("Subclass must implement this property")
121 def descriptor_file(self
):
122 """ The descriptor file name (with prefix) """
123 regex
= self
.__class
__.DESCRIPTOR_REGEX
.format(
124 descriptor_type
=self
.descriptor_type
,
128 for filename
in self
.files
:
129 if re
.match(regex
, filename
):
130 if desc_file
is not None:
131 raise PackageError("Package contains more than one descriptor")
134 if desc_file
is None:
135 raise PackageError("Could not find descriptor file in package")
140 def descriptor_msg(self
):
141 """ The proto-GI descriptor message """
142 filename
= self
.descriptor_file
143 with self
.open(filename
) as hdl
:
144 _
, ext
= os
.path
.splitext(filename
)
145 nsd
= self
.serializer
.from_file_hdl(hdl
, ext
)
149 def json_descriptor(self
):
150 """ The JSON serialized descriptor message"""
151 nsd
= self
.descriptor_msg
152 return self
.serializer
.to_json_string(nsd
)
155 def descriptor_id(self
):
156 """ The descriptor id which uniquely identifies this descriptor in the system """
157 if not self
.descriptor_msg
.has_field("id"):
158 msg
= "Descriptor must have an id field"
160 raise PackageError(msg
)
162 return self
.descriptor_msg
.id
165 def descriptor_name(self
):
166 """ The descriptor name of this descriptor in the system """
167 if not self
.descriptor_msg
.has_field("name"):
168 msg
= "Descriptor name not present"
170 raise PackageError(msg
)
172 return self
.descriptor_msg
.name
175 def descriptor_version(self
):
176 desc_msg
= self
.descriptor_msg
177 return desc_msg
.version
if desc_msg
.has_field("version") else ''
180 def descriptor_vendor(self
):
181 desc_msg
= self
.descriptor_msg
182 return desc_msg
.vendor
if desc_msg
.has_field("vendor") else ''
185 def get_descriptor_patterns(cls
):
186 """ Returns a tuple of descriptor regex and Package Types """
187 package_types
= (VnfdPackage
, NsdPackage
)
190 for pkg_cls
in package_types
:
191 regex
= cls
.DESCRIPTOR_REGEX
.format(
192 descriptor_type
=pkg_cls
.DESCRIPTOR_TYPE
,
196 patterns
.append((regex
, pkg_cls
))
201 def from_package_files(cls
, log
, open_fn
, files
):
202 """ Creates a new DescriptorPackage subclass instance from a list of files
204 This classmethod detects the Package type from the package contents
205 and returns a new Package instance.
207 This will NOT subsequently add the files to the package so that must
208 be done by the client
212 open_fn - A function which can take a file name and mode and return
214 files - A list of files which would be added to the package after
218 A new DescriptorPackage subclass of the correct type for the descriptor
221 PackageError - Package type could not be determined from the list of files.
223 patterns
= cls
.get_descriptor_patterns()
227 for regex
, cls
in patterns
:
229 if re
.match(regex
, name
) is not None:
234 log
.error("No file in archive matched known descriptor formats: %s", regexes
)
235 raise PackageError("Could not determine package type from contents")
237 package
= pkg_cls(log
, open_fn
)
241 def from_descriptor_file_hdl(cls
, log
, file_hdl
):
242 """ Creates a new DescriptorPackage from a descriptor file handle
244 The descriptor file is added to the package before returning.
248 file_hdl - A file handle whose name attribute can be recognized as
249 particular descriptor type.
252 A new DescriptorPackage subclass of the correct type for the descriptor
255 PackageError - Package type could not be determined from the list of files.
256 ValueError - file_hdl did not have a name attribute provided
259 package_types
= (VnfdPackage
, NsdPackage
)
260 filename_patterns
= []
261 for package_cls
in package_types
:
262 filename_patterns
.append(
263 (r
".*{}.*".format(package_cls
.DESCRIPTOR_TYPE
), package_cls
)
266 if not hasattr(file_hdl
, 'name'):
267 raise ValueError("File descriptor must have a name attribute to create a descriptor package")
269 # Iterate through the recognized patterns and assign files accordingly
271 for pattern
, cls
in filename_patterns
:
272 if re
.match(pattern
, file_hdl
.name
):
277 raise PackageError("Could not determine package type from file name: %s" % file_hdl
.name
)
279 _
, ext
= os
.path
.splitext(file_hdl
.name
)
281 package_cls
.SERIALIZER
.from_file_hdl(file_hdl
, ext
)
282 except convert
.SerializationError
as e
:
283 raise PackageError("Could not deserialize descriptor %s" % file_hdl
.name
) from e
285 # Create a new file handle for each open call to prevent independent clients
286 # from affecting each other
288 new_hdl
= io
.BytesIO(file_hdl
.read())
290 def do_open(file_path
):
291 assert file_path
== file_hdl
.name
292 hdl
= io
.BytesIO(new_hdl
.getvalue())
295 desc_pkg
= package_cls(log
, do_open
)
296 desc_pkg
.add_file(file_hdl
.name
)
300 def get_file_mode(self
, pkg_file
):
301 """ Returns the file mode for the package file
304 pkg_file - A file name in the package
310 PackageError - The file does not exist in the package
313 return self
._package
_file
_mode
_map
[pkg_file
]
314 except KeyError as e
:
315 msg
= "Could not find package_file: %s" % pkg_file
317 raise PackageError(msg
) from e
319 def extract_dir(self
, src_dir
, dest_root_dir
, extract_images
=False):
320 """ Extract a specific directory contents to dest_root_dir
323 src_dir - A directory within the package (None means all files/directories)
324 dest_root_dir - A directory to extract directory contents to
325 extract_images - A flag indicating whether we want to extract images
328 ExtractError - Directory contents could not be extracted
330 if src_dir
is not None and src_dir
not in self
._package
_dirs
:
331 raise ExtractError("Could not find source dir: %s" % src_dir
)
333 for filename
in self
.files
:
334 if not extract_images
and image
.is_image_file(filename
):
337 if src_dir
is not None and not filename
.startswith(src_dir
):
340 # Copy the contents of the file to the correct path
341 # Remove the common prefix and create the dest filename
342 if src_dir
is not None:
343 fname
= filename
[len(src_dir
):]
348 dest_file_path
= os
.path
.join(dest_root_dir
, fname
)
349 dest_dir_path
= os
.path
.dirname(dest_file_path
)
350 if not os
.path
.exists(dest_dir_path
):
351 os
.makedirs(dest_dir_path
)
353 with
open(dest_file_path
, 'wb') as dst_hdl
:
354 with self
.open(filename
) as src_hdl
:
355 shutil
.copyfileobj(src_hdl
, dst_hdl
, 10 * 1024 * 1024)
357 # Set the file mode to original
358 os
.chmod(dest_file_path
, self
._package
_file
_mode
_map
[filename
])
360 def insert_file(self
, new_file
, dest_file
, rel_path
, mode
=0o777):
361 self
.add_file(rel_path
, mode
)
364 # Copy the contents of the file to the correct path
365 # For folder creation (or nested folders), dest_file appears w/ trailing "/" like: dir1/ or dir1/dir2/
366 # For regular file upload, dest_file appears as dir1/abc.txt
368 dest_dir_path
= os
.path
.dirname(dest_file
)
369 if not os
.path
.isdir(dest_dir_path
):
370 os
.makedirs(dest_dir_path
)
371 if not os
.path
.basename(dest_file
):
372 self
._log
.debug("Created dir path, no filename to insert in {}, skipping..".format(dest_dir_path
))
375 with
open(dest_file
, 'wb') as dst_hdl
:
376 with
open(new_file
, 'rb') as src_hdl
:
377 shutil
.copyfileobj(src_hdl
, dst_hdl
, 10 * 1024 * 1024)
379 # Set the file mode to original
380 os
.chmod(dest_file
, self
._package
_file
_mode
_map
[rel_path
])
381 except Exception as e
:
382 # Clear the file when an exception happens
383 if os
.path
.isfile(dest_file
):
386 raise PackageAppendError(str(e
))
388 def delete_file(self
, dest_file
, rel_path
):
389 self
.remove_file(rel_path
)
393 except Exception as e
:
394 raise PackageAppendError(str(e
))
396 def extract_file(self
, src_file
, dest_file
):
397 """ Extract a specific package file to dest_file
399 The destination directory will be created if it does not exist.
402 src_file - A file within the package
403 dest_file - A file path to extract file contents to
406 ExtractError - Directory contents could not be extracted
408 if src_file
not in self
._package
_file
_mode
_map
:
409 msg
= "Could not find source file %s" % src_file
411 raise ExtractError(msg
)
413 # Copy the contents of the file to the correct path
414 dest_dir_path
= os
.path
.dirname(dest_file
)
415 if not os
.path
.isdir(dest_dir_path
):
416 os
.makedirs(dest_dir_path
)
418 with
open(dest_file
, 'wb') as dst_hdl
:
419 with self
.open(src_file
) as src_hdl
:
420 shutil
.copyfileobj(src_hdl
, dst_hdl
, 10 * 1024 * 1024)
422 # Set the file mode to original
423 os
.chmod(dest_file
, self
._package
_file
_mode
_map
[src_file
])
425 def extract(self
, dest_root_dir
, extract_images
=False):
426 """ Extract all package contents to a destination directory
429 dest_root_dir - The directory to extract package contents to
432 NotADirectoryError - dest_root_dir is not a directory
435 """ Find comon prefix of all files in package. This prefix will be
436 used to collapse directory structure during extraction to eliminate
437 empty nested folders.
441 common_dir
.add(os
.path
.dirname(f
))
442 prefix
= os
.path
.commonprefix(list(common_dir
))
443 return prefix
if prefix
else None
445 if not os
.path
.isdir(dest_root_dir
):
446 raise NotADirectoryError(dest_root_dir
)
448 self
.extract_dir(find_prefix(), dest_root_dir
, extract_images
)
450 def open(self
, rel_path
):
451 """ Open a file contained in the package in read-only, binary mode.
454 rel_path - The file path within the package
457 A file-like object opened in read-only mode.
460 PackageError - The file could not be opened
463 return self
._open
_fn
(rel_path
)
464 except Exception as e
:
465 msg
= "Could not open file from package: %s" % rel_path
466 self
._log
.warning(msg
)
467 raise PackageError(msg
) from e
469 def add_file(self
, rel_path
, mode
=0o777):
470 """ Add a file to the package.
472 The file should be specified as a relative path to the package
473 root. The open_fn provided in the constructor must be able to
474 take the relative path and open the actual source file from
475 wherever the file actually is stored.
477 If the file's parent directories do not yet exist, add them to
481 rel_path - The file path relative to the top of the package.
482 mode - The permission mode the file should be stored with so
483 it can be extracted with the correct permissions.
486 PackageError - The file could not be added to the package
489 raise PackageError("Empty file name added")
491 if rel_path
in self
._package
_file
_mode
_map
:
492 raise PackageError("File %s already exists in package" % rel_path
)
494 # If the file's directory is not in the package add it.
495 rel_dir
= os
.path
.dirname(rel_path
)
497 self
._package
_dirs
.add(rel_dir
)
498 rel_dir
= os
.path
.dirname(rel_dir
)
500 self
._package
_file
_mode
_map
[rel_path
] = mode
502 def remove_file(self
, rel_path
):
504 raise PackageError("Empty file name added")
506 if rel_path
not in self
._package
_file
_mode
_map
:
507 raise PackageError("File %s does not exist in package" % rel_path
)
509 del self
._package
_file
_mode
_map
[rel_path
]
511 def add_dir(self
, rel_path
):
512 """ Add a directory to the package
515 rel_path - The directories relative path.
518 PackageError - A file already exists in the package with the same name.
520 if rel_path
in self
._package
_file
_mode
_map
:
521 raise PackageError("File already exists with the same name: %s", rel_path
)
523 if rel_path
in self
._package
_dirs
:
524 self
._log
.warning("%s directory already exists", rel_path
)
527 self
._package
_dirs
.add(rel_path
)
530 class NsdPackage(DescriptorPackage
):
531 DESCRIPTOR_TYPE
= "nsd"
532 SERIALIZER
= convert
.RwNsdSerializer()
535 def descriptor_type(self
):
539 def serializer(self
):
540 return NsdPackage
.SERIALIZER
543 class VnfdPackage(DescriptorPackage
):
544 DESCRIPTOR_TYPE
= "vnfd"
545 SERIALIZER
= convert
.RwVnfdSerializer()
548 def descriptor_type(self
):
552 def serializer(self
):
553 return VnfdPackage
.SERIALIZER
555 class PackageConstructValidator(object):
557 def __init__(self
, log
):
560 def validate(self
, package
):
561 """ Validate presence of descriptor file (.yaml) at the top level in the
562 package folder structure.
565 package - The Descriptor Package being validated.
569 PackageValidationError - The package validation failed for some
573 desc_file
= package
.descriptor_file
574 prefix
, desc_file
= package
.prefix
.rstrip('/'), desc_file
.rstrip('/')
576 if os
.path
.dirname(desc_file
) != prefix
:
577 msg
= "Descriptor file {} not found in expcted location {}".format(desc_file
, prefix
)
579 raise PackageValidationError(msg
)
582 class PackageChecksumValidator(object):
583 """ This class uses the checksums.txt file in the package
584 and validates that all files in the package match the checksum that exists within
587 CHECKSUM_FILE
= "{prefix}checksums.txt"
589 def __init__(self
, log
):
591 self
.validated_file_checksums
= {}
594 def get_package_checksum_file(cls
, package
):
595 checksum_file
= cls
.CHECKSUM_FILE
.format(prefix
=package
.prefix
)
596 if checksum_file
not in package
.files
:
597 raise FileNotFoundError("%s does not exist in archive" % checksum_file
)
603 return self
.validated_file_checksums
605 def validate(self
, package
):
606 """ Validate file checksums match that in the checksums.txt
609 package - The Descriptor Package which possiblity contains checksums.txt
611 Returns: A dictionary of files that were validated by the checksums.txt
612 along with their checksums
615 PackageValidationError - The package validation failed for some
617 PackageFileChecksumError - A file within the package did not match the
618 checksum within checksums.txt
622 checksum_file
= PackageChecksumValidator
.get_package_checksum_file(package
)
623 with package
.open(checksum_file
) as checksum_hdl
:
624 archive_checksums
= checksums
.ArchiveChecksums
.from_file_desc(checksum_hdl
)
625 except (FileNotFoundError
, PackageError
) as e
:
626 self
._log
.warning("Could not open package checksum file. Not validating checksums.")
627 return self
.validated_file_checksums
629 for pkg_file
in package
.files
:
630 if pkg_file
== checksum_file
:
633 pkg_file_no_prefix
= pkg_file
.replace(package
.prefix
, "", 1)
634 if pkg_file_no_prefix
not in archive_checksums
:
635 self
._log
.warning("File %s not found in checksum file %s",
636 pkg_file
, checksum_file
)
640 with package
.open(pkg_file
) as pkg_file_hdl
:
641 file_checksum
= checksums
.checksum(pkg_file_hdl
)
642 except PackageError
as e
:
643 msg
= "Could not read package file {} for checksum validation: {}".format(
646 raise PackageValidationError(msg
) from e
648 if archive_checksums
[pkg_file_no_prefix
] != file_checksum
:
649 msg
= "{} checksum ({}) did match expected checksum ({})".format(
650 pkg_file
, file_checksum
, archive_checksums
[pkg_file_no_prefix
]
653 raise PackageFileChecksumError(pkg_file
)
655 self
.validated_file_checksums
[pkg_file
] = file_checksum
658 class TarPackageArchive(object):
659 """ This class represents a package stored within a tar.gz archive file """
660 def __init__(self
, log
, tar_file_hdl
, mode
="r"):
662 self
._tar
_filepath
= tar_file_hdl
665 self
._tarfile
= tarfile
.open(fileobj
=tar_file_hdl
, mode
=mode
)
670 return "TarPackageArchive(%s)" % self
._tar
_filepath
672 def _get_members(self
):
673 return [info
for info
in self
._tarfile
.getmembers()]
675 def _load_archive(self
):
676 self
._tar
_infos
= {info
.name
: info
for info
in self
._get
_members
() if info
.name
}
682 """ Close the opened tarfile"""
683 if self
._tarfile
is not None:
684 self
._tarfile
.close()
689 """ The list of file members within the tar file """
690 return [name
for name
in self
._tar
_infos
if tarfile
.TarInfo
.isfile(self
._tar
_infos
[name
])]
692 def open_file(self
, rel_file_path
):
693 """ Opens a file within the archive as read-only, byte mode.
696 rel_file_path - The file path within the archive to open
699 A file like object (see tarfile.extractfile())
702 ArchiveError - The file could not be opened for some generic reason.
704 if rel_file_path
not in self
._tar
_infos
:
705 raise ArchiveError("Could not find %s in tar file", rel_file_path
)
708 return self
._tarfile
.extractfile(rel_file_path
)
709 except tarfile
.TarError
as e
:
710 msg
= "Failed to read file {} from tarfile {}: {}".format(
711 rel_file_path
, self
._tar
_filepath
, str(e
)
714 raise ArchiveError(msg
) from e
716 def create_package(self
):
717 """ Creates a Descriptor package from the archive contents
720 A DescriptorPackage of the correct descriptor type
722 package
= DescriptorPackage
.from_package_files(self
._log
, self
.open_file
, self
.filenames
)
723 for pkg_file
in self
.filenames
:
724 package
.add_file(pkg_file
, self
._tar
_infos
[pkg_file
].mode
)
729 class TemporaryPackage(object):
730 """ This class is a container for a temporary file-backed package
732 This class contains a DescriptorPackage and can be used in place of one.
733 Provides a useful context manager which will close and destroy the file
734 that is backing the DescriptorPackage on exit.
736 def __init__(self
, log
, package
, file_hdl
):
738 self
._package
= package
739 self
._file
_hdl
= file_hdl
741 if not hasattr(self
._file
_hdl
, "name"):
742 raise ValueError("File handle must have a name attribute")
744 def __getattr__(self
, attr
):
745 return getattr(self
._package
, attr
)
750 def __exit__(self
, type, value
, tb
):
754 """ Returns the filepath with is backing the Package """
755 return self
._file
_hdl
.name
758 """ The contained DescriptorPackage instance """
762 """ Close and remove the backed file """
763 filename
= self
._file
_hdl
.name
766 self
._file
_hdl
.close()
768 self
._log
.warning("Failed to close package file: %s", str(e
))
773 self
._log
.warning("Failed to remove package file: %s", str(e
))