3 # Copyright 2016 RIFT.IO Inc
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
24 from . import checksums
29 class ArchiveError(Exception):
33 class ExtractError(Exception):
37 class PackageError(Exception):
41 class PackageValidationError(Exception):
45 class PackageFileChecksumError(PackageValidationError
):
46 def __init__(self
, filename
):
47 self
.filename
= filename
48 super().__init
__("Checksum mismatch for {}".format(filename
))
51 class DescriptorPackage(object):
52 """ This class provides an base class for a descriptor package representing
54 A descriptor package is a package which contains a single descriptor and any
55 associated files (logos, charms, scripts, etc). This package representation
56 attempts to be agnostic as to where the package files are being stored
57 (in memory, on disk, etc).
59 The package provides a simple interface to interact with the files within the
60 package and access the contained descriptor.
62 DESCRIPTOR_REGEX
= r
"{prefix}({descriptor_type}/[^/]*|[^/]*{descriptor_type})\.(xml|yml|yaml|json)$"
64 def __init__(self
, log
, open_fn
):
66 self
._open
_fn
= open_fn
68 self
._package
_file
_mode
_map
= {}
69 self
._package
_dirs
= set()
73 """ Return the leading parent directories shared by all files in the package
75 In order to remain flexible as to where tar was invoked to create the package,
76 the prefix represents the common parent directory path which all files in the
77 package have in common.
79 entries
= list(self
._package
_file
_mode
_map
) + list(self
._package
_dirs
)
82 prefix
= os
.path
.commonprefix(entries
)
83 if prefix
and not prefix
.endswith("/"):
85 elif len(entries
) == 1:
88 prefix
= os
.path
.dirname(entry
) + "/"
98 """ Return all files (with the prefix) in the package """
99 return list(self
._package
_file
_mode
_map
)
103 """ Return all directories in the package """
104 return list(self
._package
_dirs
)
107 def descriptor_type(self
):
108 """ A shorthand name for the type of descriptor (e.g. nsd)"""
109 raise NotImplementedError("Subclass must implement this property")
112 def serializer(self
):
113 """ An instance of convert.ProtoMessageSerializer """
114 raise NotImplementedError("Subclass must implement this property")
117 def descriptor_file(self
):
118 """ The descriptor file name (with prefix) """
119 regex
= self
.__class
__.DESCRIPTOR_REGEX
.format(
120 descriptor_type
=self
.descriptor_type
,
124 for filename
in self
.files
:
125 if re
.match(regex
, filename
):
126 if desc_file
is not None:
127 raise PackageError("Package contains more than one descriptor")
130 if desc_file
is None:
131 raise PackageError("Could not find descriptor file in package")
136 def descriptor_msg(self
):
137 """ The proto-GI descriptor message """
138 filename
= self
.descriptor_file
139 with self
.open(filename
) as hdl
:
140 _
, ext
= os
.path
.splitext(filename
)
141 nsd
= self
.serializer
.from_file_hdl(hdl
, ext
)
145 def json_descriptor(self
):
146 """ The JSON serialized descriptor message"""
147 nsd
= self
.descriptor_msg
148 return self
.serializer
.to_json_string(nsd
)
151 def descriptor_id(self
):
152 """ The descriptor id which uniquely identifies this descriptor in the system """
153 if not self
.descriptor_msg
.has_field("id"):
154 msg
= "Descriptor must have an id field"
156 raise PackageError(msg
)
158 return self
.descriptor_msg
.id
161 def get_descriptor_patterns(cls
):
162 """ Returns a tuple of descriptor regex and Package Types """
163 package_types
= (VnfdPackage
, NsdPackage
)
166 for pkg_cls
in package_types
:
167 regex
= cls
.DESCRIPTOR_REGEX
.format(
168 descriptor_type
=pkg_cls
.DESCRIPTOR_TYPE
,
172 patterns
.append((regex
, pkg_cls
))
177 def from_package_files(cls
, log
, open_fn
, files
):
178 """ Creates a new DescriptorPackage subclass instance from a list of files
180 This classmethod detects the Package type from the package contents
181 and returns a new Package instance.
183 This will NOT subsequently add the files to the package so that must
184 be done by the client
188 open_fn - A function which can take a file name and mode and return
190 files - A list of files which would be added to the package after
194 A new DescriptorPackage subclass of the correct type for the descriptor
197 PackageError - Package type could not be determined from the list of files.
199 patterns
= cls
.get_descriptor_patterns()
203 for regex
, cls
in patterns
:
205 if re
.match(regex
, name
) is not None:
210 log
.error("No file in archive matched known descriptor formats: %s", regexes
)
211 raise PackageError("Could not determine package type from contents")
213 package
= pkg_cls(log
, open_fn
)
217 def from_descriptor_file_hdl(cls
, log
, file_hdl
):
218 """ Creates a new DescriptorPackage from a descriptor file handle
220 The descriptor file is added to the package before returning.
224 file_hdl - A file handle whose name attribute can be recognized as
225 particular descriptor type.
228 A new DescriptorPackage subclass of the correct type for the descriptor
231 PackageError - Package type could not be determined from the list of files.
232 ValueError - file_hdl did not have a name attribute provided
235 package_types
= (VnfdPackage
, NsdPackage
)
236 filename_patterns
= []
237 for package_cls
in package_types
:
238 filename_patterns
.append(
239 (r
".*{}.*".format(package_cls
.DESCRIPTOR_TYPE
), package_cls
)
242 if not hasattr(file_hdl
, 'name'):
243 raise ValueError("File descriptor must have a name attribute to create a descriptor package")
245 # Iterate through the recognized patterns and assign files accordingly
247 for pattern
, cls
in filename_patterns
:
248 if re
.match(pattern
, file_hdl
.name
):
253 raise PackageError("Could not determine package type from file name: %s" % file_hdl
.name
)
255 _
, ext
= os
.path
.splitext(file_hdl
.name
)
257 package_cls
.SERIALIZER
.from_file_hdl(file_hdl
, ext
)
258 except convert
.SerializationError
as e
:
259 raise PackageError("Could not deserialize descriptor %s" % file_hdl
.name
) from e
261 # Create a new file handle for each open call to prevent independent clients
262 # from affecting each other
264 new_hdl
= io
.BytesIO(file_hdl
.read())
266 def do_open(file_path
):
267 assert file_path
== file_hdl
.name
268 hdl
= io
.BytesIO(new_hdl
.getvalue())
271 desc_pkg
= package_cls(log
, do_open
)
272 desc_pkg
.add_file(file_hdl
.name
)
276 def get_file_mode(self
, pkg_file
):
277 """ Returns the file mode for the package file
280 pkg_file - A file name in the package
286 PackageError - The file does not exist in the package
289 return self
._package
_file
_mode
_map
[pkg_file
]
290 except KeyError as e
:
291 msg
= "Could not find package_file: %s" % pkg_file
293 raise PackageError(msg
) from e
295 def extract_dir(self
, src_dir
, dest_root_dir
, extract_images
=False):
296 """ Extract a specific directory contents to dest_root_dir
299 src_dir - A directory within the package (None means all files/directories)
300 dest_root_dir - A directory to extract directory contents to
301 extract_images - A flag indicating whether we want to extract images
304 ExtractError - Directory contents could not be extracted
306 if src_dir
is not None and src_dir
not in self
._package
_dirs
:
307 raise ExtractError("Could not find source dir: %s" % src_dir
)
309 for filename
in self
.files
:
310 if not extract_images
and image
.is_image_file(filename
):
313 if src_dir
is not None and not filename
.startswith(src_dir
):
316 # Copy the contents of the file to the correct path
317 # Remove the common prefix and create the dest filename
318 if src_dir
is not None:
319 fname
= filename
[len(src_dir
):]
324 dest_file_path
= os
.path
.join(dest_root_dir
, fname
)
325 dest_dir_path
= os
.path
.dirname(dest_file_path
)
326 if not os
.path
.exists(dest_dir_path
):
327 os
.makedirs(dest_dir_path
)
329 with
open(dest_file_path
, 'wb') as dst_hdl
:
330 with self
.open(filename
) as src_hdl
:
331 shutil
.copyfileobj(src_hdl
, dst_hdl
, 10 * 1024 * 1024)
333 # Set the file mode to original
334 os
.chmod(dest_file_path
, self
._package
_file
_mode
_map
[filename
])
336 def extract_file(self
, src_file
, dest_file
):
337 """ Extract a specific package file to dest_file
339 The destination directory will be created if it does not exist.
342 src_file - A file within the package
343 dest_file - A file path to extract file contents to
346 ExtractError - Directory contents could not be extracted
348 if src_file
not in self
._package
_file
_mode
_map
:
349 msg
= "Could not find source file %s" % src_file
351 raise ExtractError(msg
)
353 # Copy the contents of the file to the correct path
354 dest_dir_path
= os
.path
.dirname(dest_file
)
355 if not os
.path
.isdir(dest_dir_path
):
356 os
.makedirs(dest_dir_path
)
358 with
open(dest_file
, 'wb') as dst_hdl
:
359 with self
.open(src_file
) as src_hdl
:
360 shutil
.copyfileobj(src_hdl
, dst_hdl
, 10 * 1024 * 1024)
362 # Set the file mode to original
363 os
.chmod(dest_file
, self
._package
_file
_mode
_map
[src_file
])
365 def extract(self
, dest_root_dir
, extract_images
=False):
366 """ Extract all package contents to a destination directory
369 dest_root_dir - The directory to extract package contents to
372 NotADirectoryError - dest_root_dir is not a directory
374 if not os
.path
.isdir(dest_root_dir
):
375 raise NotADirectoryError(dest_root_dir
)
377 self
.extract_dir(None, dest_root_dir
, extract_images
)
379 def open(self
, rel_path
):
380 """ Open a file contained in the package in read-only, binary mode.
383 rel_path - The file path within the package
386 A file-like object opened in read-only mode.
389 PackageError - The file could not be opened
392 return self
._open
_fn
(rel_path
)
393 except Exception as e
:
394 msg
= "Could not open file from package: %s" % rel_path
395 self
._log
.warning(msg
)
396 raise PackageError(msg
) from e
398 def add_file(self
, rel_path
, mode
=0o777):
399 """ Add a file to the package.
401 The file should be specified as a relative path to the package
402 root. The open_fn provided in the constructor must be able to
403 take the relative path and open the actual source file from
404 wherever the file actually is stored.
406 If the file's parent directories do not yet exist, add them to
410 rel_path - The file path relative to the top of the package.
411 mode - The permission mode the file should be stored with so
412 it can be extracted with the correct permissions.
415 PackageError - The file could not be added to the package
418 raise PackageError("Empty file name added")
420 if rel_path
in self
._package
_file
_mode
_map
:
421 raise PackageError("File %s already exists in package" % rel_path
)
423 # If the file's directory is not in the package add it.
424 rel_dir
= os
.path
.dirname(rel_path
)
426 self
._package
_dirs
.add(rel_dir
)
427 rel_dir
= os
.path
.dirname(rel_dir
)
429 self
._package
_file
_mode
_map
[rel_path
] = mode
431 def add_dir(self
, rel_path
):
432 """ Add a directory to the package
435 rel_path - The directories relative path.
438 PackageError - A file already exists in the package with the same name.
440 if rel_path
in self
._package
_file
_mode
_map
:
441 raise PackageError("File already exists with the same name: %s", rel_path
)
443 if rel_path
in self
._package
_dirs
:
444 self
._log
.warning("%s directory already exists", rel_path
)
447 self
._package
_dirs
.add(rel_path
)
450 class NsdPackage(DescriptorPackage
):
451 DESCRIPTOR_TYPE
= "nsd"
452 SERIALIZER
= convert
.RwNsdSerializer()
455 def descriptor_type(self
):
459 def serializer(self
):
460 return NsdPackage
.SERIALIZER
463 class VnfdPackage(DescriptorPackage
):
464 DESCRIPTOR_TYPE
= "vnfd"
465 SERIALIZER
= convert
.RwVnfdSerializer()
468 def descriptor_type(self
):
472 def serializer(self
):
473 return VnfdPackage
.SERIALIZER
476 class PackageChecksumValidator(object):
477 """ This class uses the checksums.txt file in the package
478 and validates that all files in the package match the checksum that exists within
481 CHECKSUM_FILE
= "{prefix}checksums.txt"
483 def __init__(self
, log
):
487 def get_package_checksum_file(cls
, package
):
488 checksum_file
= cls
.CHECKSUM_FILE
.format(prefix
=package
.prefix
)
489 if checksum_file
not in package
.files
:
490 raise FileNotFoundError("%s does not exist in archive" % checksum_file
)
494 def validate(self
, package
):
495 """ Validate file checksums match that in the checksums.txt
498 package - The Descriptor Package which possiblity contains checksums.txt
500 Returns: A dictionary of files that were validated by the checksums.txt
501 along with their checksums
504 PackageValidationError - The package validation failed for some
506 PackageFileChecksumError - A file within the package did not match the
507 checksum within checksums.txt
509 validated_file_checksums
= {}
512 checksum_file
= PackageChecksumValidator
.get_package_checksum_file(package
)
513 with package
.open(checksum_file
) as checksum_hdl
:
514 archive_checksums
= checksums
.ArchiveChecksums
.from_file_desc(checksum_hdl
)
515 except (FileNotFoundError
, PackageError
) as e
:
516 self
._log
.warning("Could not open package checksum file. Not validating checksums.")
517 return validated_file_checksums
519 for pkg_file
in package
.files
:
520 if pkg_file
== checksum_file
:
523 pkg_file_no_prefix
= pkg_file
.replace(package
.prefix
, "", 1)
524 if pkg_file_no_prefix
not in archive_checksums
:
525 self
._log
.warning("File %s not found in checksum file %s",
526 pkg_file
, checksum_file
)
530 with package
.open(pkg_file
) as pkg_file_hdl
:
531 file_checksum
= checksums
.checksum(pkg_file_hdl
)
532 except PackageError
as e
:
533 msg
= "Could not read package file {} for checksum validation: {}".format(
536 raise PackageValidationError(msg
) from e
538 if archive_checksums
[pkg_file_no_prefix
] != file_checksum
:
539 msg
= "{} checksum ({}) did match expected checksum ({})".format(
540 pkg_file
, file_checksum
, archive_checksums
[pkg_file_no_prefix
]
543 raise PackageFileChecksumError(pkg_file
)
545 validated_file_checksums
[pkg_file
] = file_checksum
547 return validated_file_checksums
550 class TarPackageArchive(object):
551 """ This class represents a package stored within a tar.gz archive file """
552 def __init__(self
, log
, tar_file_hdl
, mode
="r"):
554 self
._tar
_filepath
= tar_file_hdl
557 self
._tarfile
= tarfile
.open(fileobj
=tar_file_hdl
, mode
=mode
)
562 return "TarPackageArchive(%s)" % self
._tar
_filepath
564 def _get_members(self
):
565 return [info
for info
in self
._tarfile
.getmembers()]
567 def _load_archive(self
):
568 self
._tar
_infos
= {info
.name
: info
for info
in self
._get
_members
() if info
.name
}
574 """ Close the opened tarfile"""
575 if self
._tarfile
is not None:
576 self
._tarfile
.close()
581 """ The list of file members within the tar file """
582 return [name
for name
in self
._tar
_infos
if tarfile
.TarInfo
.isfile(self
._tar
_infos
[name
])]
584 def open_file(self
, rel_file_path
):
585 """ Opens a file within the archive as read-only, byte mode.
588 rel_file_path - The file path within the archive to open
591 A file like object (see tarfile.extractfile())
594 ArchiveError - The file could not be opened for some generic reason.
596 if rel_file_path
not in self
._tar
_infos
:
597 raise ArchiveError("Could not find %s in tar file", rel_file_path
)
600 return self
._tarfile
.extractfile(rel_file_path
)
601 except tarfile
.TarError
as e
:
602 msg
= "Failed to read file {} from tarfile {}: {}".format(
603 rel_file_path
, self
._tar
_filepath
, str(e
)
606 raise ArchiveError(msg
) from e
608 def create_package(self
):
609 """ Creates a Descriptor package from the archive contents
612 A DescriptorPackage of the correct descriptor type
614 package
= DescriptorPackage
.from_package_files(self
._log
, self
.open_file
, self
.filenames
)
615 for pkg_file
in self
.filenames
:
616 package
.add_file(pkg_file
, self
._tar
_infos
[pkg_file
].mode
)
621 class TemporaryPackage(object):
622 """ This class is a container for a temporary file-backed package
624 This class contains a DescriptorPackage and can be used in place of one.
625 Provides a useful context manager which will close and destroy the file
626 that is backing the DescriptorPackage on exit.
628 def __init__(self
, log
, package
, file_hdl
):
630 self
._package
= package
631 self
._file
_hdl
= file_hdl
633 if not hasattr(self
._file
_hdl
, "name"):
634 raise ValueError("File handle must have a name attribute")
636 def __getattr__(self
, attr
):
637 return getattr(self
._package
, attr
)
642 def __exit__(self
, type, value
, tb
):
646 """ Returns the filepath with is backing the Package """
647 return self
._file
_hdl
.name
650 """ The contained DescriptorPackage instance """
654 """ Close and remove the backed file """
655 filename
= self
._file
_hdl
.name
658 self
._file
_hdl
.close()
660 self
._log
.warning("Failed to close package file: %s", str(e
))
665 self
._log
.warning("Failed to remove package file: %s", str(e
))