3 # Copyright 2016 RIFT.IO Inc
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
24 from . import checksums
29 class ArchiveError(Exception):
33 class ExtractError(Exception):
37 class PackageError(Exception):
41 class PackageValidationError(Exception):
45 class PackageFileChecksumError(PackageValidationError
):
46 def __init__(self
, filename
):
47 self
.filename
= filename
48 super().__init
__("Checksum mismatch for {}".format(filename
))
51 class DescriptorPackage(object):
52 """ This class provides an base class for a descriptor package representing
54 A descriptor package is a package which contains a single descriptor and any
55 associated files (logos, charms, scripts, etc). This package representation
56 attempts to be agnostic as to where the package files are being stored
57 (in memory, on disk, etc).
59 The package provides a simple interface to interact with the files within the
60 package and access the contained descriptor.
62 DESCRIPTOR_REGEX
= r
"{prefix}({descriptor_type}/[^/]*|[^/]*{descriptor_type})\.(xml|yml|yaml|json)$"
64 def __init__(self
, log
, open_fn
):
66 self
._open
_fn
= open_fn
68 self
._package
_file
_mode
_map
= {}
69 self
._package
_dirs
= set()
73 """ Return the leading parent directories shared by all files in the package
75 In order to remain flexible as to where tar was invoked to create the package,
76 the prefix represents the common parent directory path which all files in the
77 package have in common.
79 entries
= list(self
._package
_file
_mode
_map
) + list(self
._package
_dirs
)
82 prefix
= os
.path
.commonprefix(entries
)
83 if prefix
and not prefix
.endswith("/"):
85 elif len(entries
) == 1:
88 prefix
= os
.path
.dirname(entry
) + "/"
98 """ Return all files (with the prefix) in the package """
99 return list(self
._package
_file
_mode
_map
)
103 """ Return all directories in the package """
104 return list(self
._package
_dirs
)
107 def descriptor_type(self
):
108 """ A shorthand name for the type of descriptor (e.g. nsd)"""
109 raise NotImplementedError("Subclass must implement this property")
112 def serializer(self
):
113 """ An instance of convert.ProtoMessageSerializer """
114 raise NotImplementedError("Subclass must implement this property")
117 def descriptor_file(self
):
118 """ The descriptor file name (with prefix) """
119 regex
= self
.__class
__.DESCRIPTOR_REGEX
.format(
120 descriptor_type
=self
.descriptor_type
,
124 for filename
in self
.files
:
125 if re
.match(regex
, filename
):
126 if desc_file
is not None:
127 raise PackageError("Package contains more than one descriptor")
130 if desc_file
is None:
131 raise PackageError("Could not find descriptor file in package")
136 def descriptor_msg(self
):
137 """ The proto-GI descriptor message """
138 filename
= self
.descriptor_file
139 with self
.open(filename
) as hdl
:
140 _
, ext
= os
.path
.splitext(filename
)
141 nsd
= self
.serializer
.from_file_hdl(hdl
, ext
)
145 def json_descriptor(self
):
146 """ The JSON serialized descriptor message"""
147 nsd
= self
.descriptor_msg
148 return self
.serializer
.to_json_string(nsd
)
151 def descriptor_id(self
):
152 """ The descriptor id which uniquely identifies this descriptor in the system """
153 if not self
.descriptor_msg
.has_field("id"):
154 msg
= "Descriptor must have an id field"
156 raise PackageError(msg
)
158 return self
.descriptor_msg
.id
161 def get_descriptor_patterns(cls
):
162 """ Returns a tuple of descriptor regex and Package Types """
163 package_types
= (VnfdPackage
, NsdPackage
)
166 for pkg_cls
in package_types
:
167 regex
= cls
.DESCRIPTOR_REGEX
.format(
168 descriptor_type
=pkg_cls
.DESCRIPTOR_TYPE
,
172 patterns
.append((regex
, pkg_cls
))
177 def from_package_files(cls
, log
, open_fn
, files
):
178 """ Creates a new DescriptorPackage subclass instance from a list of files
180 This classmethod detects the Package type from the package contents
181 and returns a new Package instance.
183 This will NOT subsequently add the files to the package so that must
184 be done by the client
188 open_fn - A function which can take a file name and mode and return
190 files - A list of files which would be added to the package after
194 A new DescriptorPackage subclass of the correct type for the descriptor
197 PackageError - Package type could not be determined from the list of files.
199 patterns
= cls
.get_descriptor_patterns()
203 for regex
, cls
in patterns
:
205 if re
.match(regex
, name
) is not None:
210 log
.error("No file in archive matched known descriptor formats: %s", regexes
)
211 raise PackageError("Could not determine package type from contents")
213 package
= pkg_cls(log
, open_fn
)
217 def from_descriptor_file_hdl(cls
, log
, file_hdl
):
218 """ Creates a new DescriptorPackage from a descriptor file handle
220 The descriptor file is added to the package before returning.
224 file_hdl - A file handle whose name attribute can be recognized as
225 particular descriptor type.
228 A new DescriptorPackage subclass of the correct type for the descriptor
231 PackageError - Package type could not be determined from the list of files.
232 ValueError - file_hdl did not have a name attribute provided
235 package_types
= (VnfdPackage
, NsdPackage
)
236 filename_patterns
= []
237 for package_cls
in package_types
:
238 filename_patterns
.append(
239 (r
".*{}.*".format(package_cls
.DESCRIPTOR_TYPE
), package_cls
)
242 if not hasattr(file_hdl
, 'name'):
243 raise ValueError("File descriptor must have a name attribute to create a descriptor package")
245 # Iterate through the recognized patterns and assign files accordingly
247 for pattern
, cls
in filename_patterns
:
248 if re
.match(pattern
, file_hdl
.name
):
253 raise PackageError("Could not determine package type from file name: %s" % file_hdl
.name
)
255 _
, ext
= os
.path
.splitext(file_hdl
.name
)
257 package_cls
.SERIALIZER
.from_file_hdl(file_hdl
, ext
)
258 except convert
.SerializationError
as e
:
259 raise PackageError("Could not deserialize descriptor %s" % file_hdl
.name
) from e
261 # Create a new file handle for each open call to prevent independent clients
262 # from affecting each other
264 new_hdl
= io
.BytesIO(file_hdl
.read())
266 def do_open(file_path
):
267 assert file_path
== file_hdl
.name
268 hdl
= io
.BytesIO(new_hdl
.getvalue())
271 desc_pkg
= package_cls(log
, do_open
)
272 desc_pkg
.add_file(file_hdl
.name
)
276 def get_file_mode(self
, pkg_file
):
277 """ Returns the file mode for the package file
280 pkg_file - A file name in the package
286 PackageError - The file does not exist in the package
289 return self
._package
_file
_mode
_map
[pkg_file
]
290 except KeyError as e
:
291 msg
= "Could not find package_file: %s" % pkg_file
293 raise PackageError(msg
) from e
295 def extract_dir(self
, src_dir
, dest_root_dir
, extract_images
=False):
296 """ Extract a specific directory contents to dest_root_dir
299 src_dir - A directory within the package (None means all files/directories)
300 dest_root_dir - A directory to extract directory contents to
301 extract_images - A flag indicating whether we want to extract images
304 ExtractError - Directory contents could not be extracted
306 if src_dir
is not None and src_dir
not in self
._package
_dirs
:
307 raise ExtractError("Could not find source dir: %s" % src_dir
)
309 for filename
in self
.files
:
310 if not extract_images
and image
.is_image_file(filename
):
313 if src_dir
is not None and not filename
.startswith(src_dir
):
316 # Copy the contents of the file to the correct path
317 dest_file_path
= os
.path
.join(dest_root_dir
, filename
)
318 dest_dir_path
= os
.path
.dirname(dest_file_path
)
319 if not os
.path
.exists(dest_dir_path
):
320 os
.makedirs(dest_dir_path
)
322 with
open(dest_file_path
, 'wb') as dst_hdl
:
323 with self
.open(filename
) as src_hdl
:
324 shutil
.copyfileobj(src_hdl
, dst_hdl
, 10 * 1024 * 1024)
326 # Set the file mode to original
327 os
.chmod(dest_file_path
, self
._package
_file
_mode
_map
[filename
])
329 def extract_file(self
, src_file
, dest_file
):
330 """ Extract a specific package file to dest_file
332 The destination directory will be created if it does not exist.
335 src_file - A file within the package
336 dest_file - A file path to extract file contents to
339 ExtractError - Directory contents could not be extracted
341 if src_file
not in self
._package
_file
_mode
_map
:
342 msg
= "Could not find source file %s" % src_file
344 raise ExtractError(msg
)
346 # Copy the contents of the file to the correct path
347 dest_dir_path
= os
.path
.dirname(dest_file
)
348 if not os
.path
.isdir(dest_dir_path
):
349 os
.makedirs(dest_dir_path
)
351 with
open(dest_file
, 'wb') as dst_hdl
:
352 with self
.open(src_file
) as src_hdl
:
353 shutil
.copyfileobj(src_hdl
, dst_hdl
, 10 * 1024 * 1024)
355 # Set the file mode to original
356 os
.chmod(dest_file
, self
._package
_file
_mode
_map
[src_file
])
358 def extract(self
, dest_root_dir
, extract_images
=False):
359 """ Extract all package contents to a destination directory
362 dest_root_dir - The directory to extract package contents to
365 NotADirectoryError - dest_root_dir is not a directory
367 if not os
.path
.isdir(dest_root_dir
):
368 raise NotADirectoryError(dest_root_dir
)
370 self
.extract_dir(None, dest_root_dir
, extract_images
)
372 def open(self
, rel_path
):
373 """ Open a file contained in the package in read-only, binary mode.
376 rel_path - The file path within the package
379 A file-like object opened in read-only mode.
382 PackageError - The file could not be opened
385 return self
._open
_fn
(rel_path
)
386 except Exception as e
:
387 msg
= "Could not open file from package: %s" % rel_path
388 self
._log
.warning(msg
)
389 raise PackageError(msg
) from e
391 def add_file(self
, rel_path
, mode
=0o777):
392 """ Add a file to the package.
394 The file should be specified as a relative path to the package
395 root. The open_fn provided in the constructor must be able to
396 take the relative path and open the actual source file from
397 wherever the file actually is stored.
399 If the file's parent directories do not yet exist, add them to
403 rel_path - The file path relative to the top of the package.
404 mode - The permission mode the file should be stored with so
405 it can be extracted with the correct permissions.
408 PackageError - The file could not be added to the package
411 raise PackageError("Empty file name added")
413 if rel_path
in self
._package
_file
_mode
_map
:
414 raise PackageError("File %s already exists in package" % rel_path
)
416 # If the file's directory is not in the package add it.
417 rel_dir
= os
.path
.dirname(rel_path
)
419 self
._package
_dirs
.add(rel_dir
)
420 rel_dir
= os
.path
.dirname(rel_dir
)
422 self
._package
_file
_mode
_map
[rel_path
] = mode
424 def add_dir(self
, rel_path
):
425 """ Add a directory to the package
428 rel_path - The directories relative path.
431 PackageError - A file already exists in the package with the same name.
433 if rel_path
in self
._package
_file
_mode
_map
:
434 raise PackageError("File already exists with the same name: %s", rel_path
)
436 if rel_path
in self
._package
_dirs
:
437 self
._log
.warning("%s directory already exists", rel_path
)
440 self
._package
_dirs
.add(rel_path
)
443 class NsdPackage(DescriptorPackage
):
444 DESCRIPTOR_TYPE
= "nsd"
445 SERIALIZER
= convert
.RwNsdSerializer()
448 def descriptor_type(self
):
452 def serializer(self
):
453 return NsdPackage
.SERIALIZER
456 class VnfdPackage(DescriptorPackage
):
457 DESCRIPTOR_TYPE
= "vnfd"
458 SERIALIZER
= convert
.RwVnfdSerializer()
461 def descriptor_type(self
):
465 def serializer(self
):
466 return VnfdPackage
.SERIALIZER
469 class PackageChecksumValidator(object):
470 """ This class uses the checksums.txt file in the package
471 and validates that all files in the package match the checksum that exists within
474 CHECKSUM_FILE
= "{prefix}checksums.txt"
476 def __init__(self
, log
):
480 def get_package_checksum_file(cls
, package
):
481 checksum_file
= cls
.CHECKSUM_FILE
.format(prefix
=package
.prefix
)
482 if checksum_file
not in package
.files
:
483 raise FileNotFoundError("%s does not exist in archive" % checksum_file
)
487 def validate(self
, package
):
488 """ Validate file checksums match that in the checksums.txt
491 package - The Descriptor Package which possiblity contains checksums.txt
493 Returns: A dictionary of files that were validated by the checksums.txt
494 along with their checksums
497 PackageValidationError - The package validation failed for some
499 PackageFileChecksumError - A file within the package did not match the
500 checksum within checksums.txt
502 validated_file_checksums
= {}
505 checksum_file
= PackageChecksumValidator
.get_package_checksum_file(package
)
506 with package
.open(checksum_file
) as checksum_hdl
:
507 archive_checksums
= checksums
.ArchiveChecksums
.from_file_desc(checksum_hdl
)
508 except (FileNotFoundError
, PackageError
) as e
:
509 self
._log
.warning("Could not open package checksum file. Not validating checksums.")
510 return validated_file_checksums
512 for pkg_file
in package
.files
:
513 if pkg_file
== checksum_file
:
516 pkg_file_no_prefix
= pkg_file
.replace(package
.prefix
, "", 1)
517 if pkg_file_no_prefix
not in archive_checksums
:
518 self
._log
.warning("File %s not found in checksum file %s",
519 pkg_file
, checksum_file
)
523 with package
.open(pkg_file
) as pkg_file_hdl
:
524 file_checksum
= checksums
.checksum(pkg_file_hdl
)
525 except PackageError
as e
:
526 msg
= "Could not read package file {} for checksum validation: {}".format(
529 raise PackageValidationError(msg
) from e
531 if archive_checksums
[pkg_file_no_prefix
] != file_checksum
:
532 msg
= "{} checksum ({}) did match expected checksum ({})".format(
533 pkg_file
, file_checksum
, archive_checksums
[pkg_file_no_prefix
]
536 raise PackageFileChecksumError(pkg_file
)
538 validated_file_checksums
[pkg_file
] = file_checksum
540 return validated_file_checksums
543 class TarPackageArchive(object):
544 """ This class represents a package stored within a tar.gz archive file """
545 def __init__(self
, log
, tar_file_hdl
, mode
="r"):
547 self
._tar
_filepath
= tar_file_hdl
550 self
._tarfile
= tarfile
.open(fileobj
=tar_file_hdl
, mode
=mode
)
555 return "TarPackageArchive(%s)" % self
._tar
_filepath
557 def _get_members(self
):
558 return [info
for info
in self
._tarfile
.getmembers()]
560 def _load_archive(self
):
561 self
._tar
_infos
= {info
.name
: info
for info
in self
._get
_members
() if info
.name
}
567 """ Close the opened tarfile"""
568 if self
._tarfile
is not None:
569 self
._tarfile
.close()
574 """ The list of file members within the tar file """
575 return [name
for name
in self
._tar
_infos
if tarfile
.TarInfo
.isfile(self
._tar
_infos
[name
])]
577 def open_file(self
, rel_file_path
):
578 """ Opens a file within the archive as read-only, byte mode.
581 rel_file_path - The file path within the archive to open
584 A file like object (see tarfile.extractfile())
587 ArchiveError - The file could not be opened for some generic reason.
589 if rel_file_path
not in self
._tar
_infos
:
590 raise ArchiveError("Could not find %s in tar file", rel_file_path
)
593 return self
._tarfile
.extractfile(rel_file_path
)
594 except tarfile
.TarError
as e
:
595 msg
= "Failed to read file {} from tarfile {}: {}".format(
596 rel_file_path
, self
._tar
_filepath
, str(e
)
599 raise ArchiveError(msg
) from e
601 def create_package(self
):
602 """ Creates a Descriptor package from the archive contents
605 A DescriptorPackage of the correct descriptor type
607 package
= DescriptorPackage
.from_package_files(self
._log
, self
.open_file
, self
.filenames
)
608 for pkg_file
in self
.filenames
:
609 package
.add_file(pkg_file
, self
._tar
_infos
[pkg_file
].mode
)
614 class TemporaryPackage(object):
615 """ This class is a container for a temporary file-backed package
617 This class contains a DescriptorPackage and can be used in place of one.
618 Provides a useful context manager which will close and destroy the file
619 that is backing the DescriptorPackage on exit.
621 def __init__(self
, log
, package
, file_hdl
):
623 self
._package
= package
624 self
._file
_hdl
= file_hdl
626 if not hasattr(self
._file
_hdl
, "name"):
627 raise ValueError("File handle must have a name attribute")
629 def __getattr__(self
, attr
):
630 return getattr(self
._package
, attr
)
635 def __exit__(self
, type, value
, tb
):
639 """ Returns the filepath with is backing the Package """
640 return self
._file
_hdl
.name
643 """ The contained DescriptorPackage instance """
647 """ Close and remove the backed file """
648 filename
= self
._file
_hdl
.name
651 self
._file
_hdl
.close()
653 self
._log
.warning("Failed to close package file: %s", str(e
))
658 self
._log
.warning("Failed to remove package file: %s", str(e
))