X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=rwlaunchpad%2Fplugins%2Frwlaunchpadtasklet%2Frift%2Fpackage%2Fpackage.py;fp=rwlaunchpad%2Fplugins%2Frwlaunchpadtasklet%2Frift%2Fpackage%2Fpackage.py;h=355b23b3fa523872feb7f3bc9c30bf8c0e4dd52c;hb=6f07e6f33f751ab4ffe624f6037f887b243bece2;hp=0000000000000000000000000000000000000000;hpb=72a563886272088feb7cb52e4aafbe6d2c580ff9;p=osm%2FSO.git diff --git a/rwlaunchpad/plugins/rwlaunchpadtasklet/rift/package/package.py b/rwlaunchpad/plugins/rwlaunchpadtasklet/rift/package/package.py new file mode 100644 index 00000000..355b23b3 --- /dev/null +++ b/rwlaunchpad/plugins/rwlaunchpadtasklet/rift/package/package.py @@ -0,0 +1,658 @@ + +# +# Copyright 2016 RIFT.IO Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import io +import os +import re +import shutil +import tarfile + +from . import checksums +from . import convert +from . import image + + +class ArchiveError(Exception): + pass + + +class ExtractError(Exception): + pass + + +class PackageError(Exception): + pass + + +class PackageValidationError(Exception): + pass + + +class PackageFileChecksumError(PackageValidationError): + def __init__(self, filename): + self.filename = filename + super().__init__("Checksum mismatch for {}".format(filename)) + + +class DescriptorPackage(object): + """ This class provides an base class for a descriptor package representing + + A descriptor package is a package which contains a single descriptor and any + associated files (logos, charms, scripts, etc). This package representation + attempts to be agnostic as to where the package files are being stored + (in memory, on disk, etc). + + The package provides a simple interface to interact with the files within the + package and access the contained descriptor. + """ + DESCRIPTOR_REGEX = r"{prefix}({descriptor_type}/[^/]*|[^/]*{descriptor_type})\.(xml|yml|yaml|json)$" + + def __init__(self, log, open_fn): + self._log = log + self._open_fn = open_fn + + self._package_file_mode_map = {} + self._package_dirs = set() + + @property + def prefix(self): + """ Return the leading parent directories shared by all files in the package + + In order to remain flexible as to where tar was invoked to create the package, + the prefix represents the common parent directory path which all files in the + package have in common. + """ + entries = list(self._package_file_mode_map) + list(self._package_dirs) + + if len(entries) > 1: + prefix = os.path.commonprefix(entries) + if prefix and not prefix.endswith("/"): + prefix += "/" + elif len(entries) == 1: + entry = entries[0] + if "/" in entry: + prefix = os.path.dirname(entry) + "/" + else: + prefix = "" + else: + prefix = "" + + return prefix + + @property + def files(self): + """ Return all files (with the prefix) in the package """ + return list(self._package_file_mode_map) + + @property + def dirs(self): + """ Return all directories in the package """ + return list(self._package_dirs) + + @property + def descriptor_type(self): + """ A shorthand name for the type of descriptor (e.g. nsd)""" + raise NotImplementedError("Subclass must implement this property") + + @property + def serializer(self): + """ An instance of convert.ProtoMessageSerializer """ + raise NotImplementedError("Subclass must implement this property") + + @property + def descriptor_file(self): + """ The descriptor file name (with prefix) """ + regex = self.__class__.DESCRIPTOR_REGEX.format( + descriptor_type=self.descriptor_type, + prefix=self.prefix, + ) + desc_file = None + for filename in self.files: + if re.match(regex, filename): + if desc_file is not None: + raise PackageError("Package contains more than one descriptor") + desc_file = filename + + if desc_file is None: + raise PackageError("Could not find descriptor file in package") + + return desc_file + + @property + def descriptor_msg(self): + """ The proto-GI descriptor message """ + filename = self.descriptor_file + with self.open(filename) as hdl: + _, ext = os.path.splitext(filename) + nsd = self.serializer.from_file_hdl(hdl, ext) + return nsd + + @property + def json_descriptor(self): + """ The JSON serialized descriptor message""" + nsd = self.descriptor_msg + return self.serializer.to_json_string(nsd) + + @property + def descriptor_id(self): + """ The descriptor id which uniquely identifies this descriptor in the system """ + if not self.descriptor_msg.has_field("id"): + msg = "Descriptor must have an id field" + self._log.error(msg) + raise PackageError(msg) + + return self.descriptor_msg.id + + @classmethod + def get_descriptor_patterns(cls): + """ Returns a tuple of descriptor regex and Package Types """ + package_types = (VnfdPackage, NsdPackage) + patterns = [] + + for pkg_cls in package_types: + regex = cls.DESCRIPTOR_REGEX.format( + descriptor_type=pkg_cls.DESCRIPTOR_TYPE, + prefix=".*" + ) + + patterns.append((regex, pkg_cls)) + + return patterns + + @classmethod + def from_package_files(cls, log, open_fn, files): + """ Creates a new DescriptorPackage subclass instance from a list of files + + This classmethod detects the Package type from the package contents + and returns a new Package instance. + + This will NOT subsequently add the files to the package so that must + be done by the client + + Arguments: + log - A logger + open_fn - A function which can take a file name and mode and return + a file handle. + files - A list of files which would be added to the package after + intantiation + + Returns: + A new DescriptorPackage subclass of the correct type for the descriptor + + Raises: + PackageError - Package type could not be determined from the list of files. + """ + patterns = cls.get_descriptor_patterns() + pkg_cls = None + regexes = set() + for name in files: + for regex, cls in patterns: + regexes.add(regex) + if re.match(regex, name) is not None: + pkg_cls = cls + break + + if pkg_cls is None: + log.error("No file in archive matched known descriptor formats: %s", regexes) + raise PackageError("Could not determine package type from contents") + + package = pkg_cls(log, open_fn) + return package + + @classmethod + def from_descriptor_file_hdl(cls, log, file_hdl): + """ Creates a new DescriptorPackage from a descriptor file handle + + The descriptor file is added to the package before returning. + + Arguments: + log - A logger + file_hdl - A file handle whose name attribute can be recognized as + particular descriptor type. + + Returns: + A new DescriptorPackage subclass of the correct type for the descriptor + + Raises: + PackageError - Package type could not be determined from the list of files. + ValueError - file_hdl did not have a name attribute provided + """ + + package_types = (VnfdPackage, NsdPackage) + filename_patterns = [] + for package_cls in package_types: + filename_patterns.append( + (r".*{}.*".format(package_cls.DESCRIPTOR_TYPE), package_cls) + ) + + if not hasattr(file_hdl, 'name'): + raise ValueError("File descriptor must have a name attribute to create a descriptor package") + + # Iterate through the recognized patterns and assign files accordingly + package_cls = None + for pattern, cls in filename_patterns: + if re.match(pattern, file_hdl.name): + package_cls = cls + break + + if not package_cls: + raise PackageError("Could not determine package type from file name: %s" % file_hdl.name) + + _, ext = os.path.splitext(file_hdl.name) + try: + package_cls.SERIALIZER.from_file_hdl(file_hdl, ext) + except convert.SerializationError as e: + raise PackageError("Could not deserialize descriptor %s" % file_hdl.name) from e + + # Create a new file handle for each open call to prevent independent clients + # from affecting each other + file_hdl.seek(0) + new_hdl = io.BytesIO(file_hdl.read()) + + def do_open(file_path): + assert file_path == file_hdl.name + hdl = io.BytesIO(new_hdl.getvalue()) + return hdl + + desc_pkg = package_cls(log, do_open) + desc_pkg.add_file(file_hdl.name) + + return desc_pkg + + def get_file_mode(self, pkg_file): + """ Returns the file mode for the package file + + Arguments: + pkg_file - A file name in the package + + Returns: + The permission mode + + Raises: + PackageError - The file does not exist in the package + """ + try: + return self._package_file_mode_map[pkg_file] + except KeyError as e: + msg = "Could not find package_file: %s" % pkg_file + self._log.error(msg) + raise PackageError(msg) from e + + def extract_dir(self, src_dir, dest_root_dir, extract_images=False): + """ Extract a specific directory contents to dest_root_dir + + Arguments: + src_dir - A directory within the package (None means all files/directories) + dest_root_dir - A directory to extract directory contents to + extract_images - A flag indicating whether we want to extract images + + Raises: + ExtractError - Directory contents could not be extracted + """ + if src_dir is not None and src_dir not in self._package_dirs: + raise ExtractError("Could not find source dir: %s" % src_dir) + + for filename in self.files: + if not extract_images and image.is_image_file(filename): + continue + + if src_dir is not None and not filename.startswith(src_dir): + continue + + # Copy the contents of the file to the correct path + dest_file_path = os.path.join(dest_root_dir, filename) + dest_dir_path = os.path.dirname(dest_file_path) + if not os.path.exists(dest_dir_path): + os.makedirs(dest_dir_path) + + with open(dest_file_path, 'wb') as dst_hdl: + with self.open(filename) as src_hdl: + shutil.copyfileobj(src_hdl, dst_hdl, 10 * 1024 * 1024) + + # Set the file mode to original + os.chmod(dest_file_path, self._package_file_mode_map[filename]) + + def extract_file(self, src_file, dest_file): + """ Extract a specific package file to dest_file + + The destination directory will be created if it does not exist. + + Arguments: + src_file - A file within the package + dest_file - A file path to extract file contents to + + Raises: + ExtractError - Directory contents could not be extracted + """ + if src_file not in self._package_file_mode_map: + msg = "Could not find source file %s" % src_file + self._log.error(msg) + raise ExtractError(msg) + + # Copy the contents of the file to the correct path + dest_dir_path = os.path.dirname(dest_file) + if not os.path.isdir(dest_dir_path): + os.makedirs(dest_dir_path) + + with open(dest_file, 'wb') as dst_hdl: + with self.open(src_file) as src_hdl: + shutil.copyfileobj(src_hdl, dst_hdl, 10 * 1024 * 1024) + + # Set the file mode to original + os.chmod(dest_file, self._package_file_mode_map[src_file]) + + def extract(self, dest_root_dir, extract_images=False): + """ Extract all package contents to a destination directory + + Arguments: + dest_root_dir - The directory to extract package contents to + + Raises: + NotADirectoryError - dest_root_dir is not a directory + """ + if not os.path.isdir(dest_root_dir): + raise NotADirectoryError(dest_root_dir) + + self.extract_dir(None, dest_root_dir, extract_images) + + def open(self, rel_path): + """ Open a file contained in the package in read-only, binary mode. + + Arguments: + rel_path - The file path within the package + + Returns: + A file-like object opened in read-only mode. + + Raises: + PackageError - The file could not be opened + """ + try: + return self._open_fn(rel_path) + except Exception as e: + msg = "Could not open file from package: %s" % rel_path + self._log.warning(msg) + raise PackageError(msg) from e + + def add_file(self, rel_path, mode=0o777): + """ Add a file to the package. + + The file should be specified as a relative path to the package + root. The open_fn provided in the constructor must be able to + take the relative path and open the actual source file from + wherever the file actually is stored. + + If the file's parent directories do not yet exist, add them to + the package. + + Arguments: + rel_path - The file path relative to the top of the package. + mode - The permission mode the file should be stored with so + it can be extracted with the correct permissions. + + Raises: + PackageError - The file could not be added to the package + """ + if not rel_path: + raise PackageError("Empty file name added") + + if rel_path in self._package_file_mode_map: + raise PackageError("File %s already exists in package" % rel_path) + + # If the file's directory is not in the package add it. + rel_dir = os.path.dirname(rel_path) + while rel_dir: + self._package_dirs.add(rel_dir) + rel_dir = os.path.dirname(rel_dir) + + self._package_file_mode_map[rel_path] = mode + + def add_dir(self, rel_path): + """ Add a directory to the package + + Arguments: + rel_path - The directories relative path. + + Raises: + PackageError - A file already exists in the package with the same name. + """ + if rel_path in self._package_file_mode_map: + raise PackageError("File already exists with the same name: %s", rel_path) + + if rel_path in self._package_dirs: + self._log.warning("%s directory already exists", rel_path) + return + + self._package_dirs.add(rel_path) + + +class NsdPackage(DescriptorPackage): + DESCRIPTOR_TYPE = "nsd" + SERIALIZER = convert.RwNsdSerializer() + + @property + def descriptor_type(self): + return "nsd" + + @property + def serializer(self): + return NsdPackage.SERIALIZER + + +class VnfdPackage(DescriptorPackage): + DESCRIPTOR_TYPE = "vnfd" + SERIALIZER = convert.RwVnfdSerializer() + + @property + def descriptor_type(self): + return "vnfd" + + @property + def serializer(self): + return VnfdPackage.SERIALIZER + + +class PackageChecksumValidator(object): + """ This class uses the checksums.txt file in the package + and validates that all files in the package match the checksum that exists within + the file. + """ + CHECKSUM_FILE = "{prefix}checksums.txt" + + def __init__(self, log): + self._log = log + + @classmethod + def get_package_checksum_file(cls, package): + checksum_file = cls.CHECKSUM_FILE.format(prefix=package.prefix) + if checksum_file not in package.files: + raise FileNotFoundError("%s does not exist in archive" % checksum_file) + + return checksum_file + + def validate(self, package): + """ Validate file checksums match that in the checksums.txt + + Arguments: + package - The Descriptor Package which possiblity contains checksums.txt + + Returns: A dictionary of files that were validated by the checksums.txt + along with their checksums + + Raises: + PackageValidationError - The package validation failed for some + generic reason. + PackageFileChecksumError - A file within the package did not match the + checksum within checksums.txt + """ + validated_file_checksums = {} + + try: + checksum_file = PackageChecksumValidator.get_package_checksum_file(package) + with package.open(checksum_file) as checksum_hdl: + archive_checksums = checksums.ArchiveChecksums.from_file_desc(checksum_hdl) + except (FileNotFoundError, PackageError) as e: + self._log.warning("Could not open package checksum file. Not validating checksums.") + return validated_file_checksums + + for pkg_file in package.files: + if pkg_file == checksum_file: + continue + + pkg_file_no_prefix = pkg_file.replace(package.prefix, "", 1) + if pkg_file_no_prefix not in archive_checksums: + self._log.warning("File %s not found in checksum file %s", + pkg_file, checksum_file) + continue + + try: + with package.open(pkg_file) as pkg_file_hdl: + file_checksum = checksums.checksum(pkg_file_hdl) + except PackageError as e: + msg = "Could not read package file {} for checksum validation: {}".format( + pkg_file, str(e)) + self._log.error(msg) + raise PackageValidationError(msg) from e + + if archive_checksums[pkg_file_no_prefix] != file_checksum: + msg = "{} checksum ({}) did match expected checksum ({})".format( + pkg_file, file_checksum, archive_checksums[pkg_file_no_prefix] + ) + self._log.error(msg) + raise PackageFileChecksumError(pkg_file) + + validated_file_checksums[pkg_file] = file_checksum + + return validated_file_checksums + + +class TarPackageArchive(object): + """ This class represents a package stored within a tar.gz archive file """ + def __init__(self, log, tar_file_hdl, mode="r"): + self._log = log + self._tar_filepath = tar_file_hdl + self._tar_infos = {} + + self._tarfile = tarfile.open(fileobj=tar_file_hdl, mode=mode) + + self._load_archive() + + def __repr__(self): + return "TarPackageArchive(%s)" % self._tar_filepath + + def _get_members(self): + return [info for info in self._tarfile.getmembers()] + + def _load_archive(self): + self._tar_infos = {info.name: info for info in self._get_members() if info.name} + + def __del__(self): + self.close() + + def close(self): + """ Close the opened tarfile""" + if self._tarfile is not None: + self._tarfile.close() + self._tarfile = None + + @property + def filenames(self): + """ The list of file members within the tar file """ + return [name for name in self._tar_infos if tarfile.TarInfo.isfile(self._tar_infos[name])] + + def open_file(self, rel_file_path): + """ Opens a file within the archive as read-only, byte mode. + + Arguments: + rel_file_path - The file path within the archive to open + + Returns: + A file like object (see tarfile.extractfile()) + + Raises: + ArchiveError - The file could not be opened for some generic reason. + """ + if rel_file_path not in self._tar_infos: + raise ArchiveError("Could not find %s in tar file", rel_file_path) + + try: + return self._tarfile.extractfile(rel_file_path) + except tarfile.TarError as e: + msg = "Failed to read file {} from tarfile {}: {}".format( + rel_file_path, self._tar_filepath, str(e) + ) + self._log.error(msg) + raise ArchiveError(msg) from e + + def create_package(self): + """ Creates a Descriptor package from the archive contents + + Returns: + A DescriptorPackage of the correct descriptor type + """ + package = DescriptorPackage.from_package_files(self._log, self.open_file, self.filenames) + for pkg_file in self.filenames: + package.add_file(pkg_file, self._tar_infos[pkg_file].mode) + + return package + + +class TemporaryPackage(object): + """ This class is a container for a temporary file-backed package + + This class contains a DescriptorPackage and can be used in place of one. + Provides a useful context manager which will close and destroy the file + that is backing the DescriptorPackage on exit. + """ + def __init__(self, log, package, file_hdl): + self._log = log + self._package = package + self._file_hdl = file_hdl + + if not hasattr(self._file_hdl, "name"): + raise ValueError("File handle must have a name attribute") + + def __getattr__(self, attr): + return getattr(self._package, attr) + + def __enter__(self): + return self._package + + def __exit__(self, type, value, tb): + self.close() + + def filename(self): + """ Returns the filepath with is backing the Package """ + return self._file_hdl.name + + def package(self): + """ The contained DescriptorPackage instance """ + return self._package + + def close(self): + """ Close and remove the backed file """ + filename = self._file_hdl.name + + try: + self._file_hdl.close() + except OSError as e: + self._log.warning("Failed to close package file: %s", str(e)) + + try: + os.remove(filename) + except OSError as e: + self._log.warning("Failed to remove package file: %s", str(e))