| |
| # |
| # Copyright 2016 RIFT.IO Inc |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| |
| import io |
| import os |
| import re |
| import shutil |
| import tarfile |
| |
| from . import checksums |
| from . import convert |
| from . import image |
| |
| |
| class ArchiveError(Exception): |
| pass |
| |
| |
| class ExtractError(Exception): |
| pass |
| |
| |
| class PackageError(Exception): |
| pass |
| |
| |
| class PackageValidationError(Exception): |
| pass |
| |
| |
| class PackageAppendError(Exception): |
| pass |
| |
| |
| class PackageFileChecksumError(PackageValidationError): |
| def __init__(self, filename): |
| self.filename = filename |
| super().__init__("Checksum mismatch for {}".format(filename)) |
| |
| |
| class DescriptorPackage(object): |
| """ This class provides an base class for a descriptor package representing |
| |
| A descriptor package is a package which contains a single descriptor and any |
| associated files (logos, charms, scripts, etc). This package representation |
| attempts to be agnostic as to where the package files are being stored |
| (in memory, on disk, etc). |
| |
| The package provides a simple interface to interact with the files within the |
| package and access the contained descriptor. |
| """ |
| DESCRIPTOR_REGEX = r"{prefix}({descriptor_type}/[^/]*|[^/]*{descriptor_type})\.(xml|yml|yaml|json)$" |
| |
| def __init__(self, log, open_fn): |
| self._log = log |
| self._open_fn = open_fn |
| |
| self._package_file_mode_map = {} |
| self._package_dirs = set() |
| |
| @property |
| def prefix(self): |
| """ Return the leading parent directories shared by all files in the package |
| |
| In order to remain flexible as to where tar was invoked to create the package, |
| the prefix represents the common parent directory path which all files in the |
| package have in common. |
| """ |
| entries = list(self._package_file_mode_map) + list(self._package_dirs) |
| |
| if len(entries) > 1: |
| prefix = os.path.commonprefix(entries) |
| if prefix and not prefix.endswith("/"): |
| prefix += "/" |
| elif len(entries) == 1: |
| entry = entries[0] |
| if "/" in entry: |
| prefix = os.path.dirname(entry) + "/" |
| else: |
| prefix = "" |
| else: |
| prefix = "" |
| |
| return prefix |
| |
| @property |
| def files(self): |
| """ Return all files (with the prefix) in the package """ |
| return list(self._package_file_mode_map) |
| |
| @property |
| def dirs(self): |
| """ Return all directories in the package """ |
| return list(self._package_dirs) |
| |
| @property |
| def descriptor_type(self): |
| """ A shorthand name for the type of descriptor (e.g. nsd)""" |
| raise NotImplementedError("Subclass must implement this property") |
| |
| @property |
| def serializer(self): |
| """ An instance of convert.ProtoMessageSerializer """ |
| raise NotImplementedError("Subclass must implement this property") |
| |
| @property |
| def descriptor_file(self): |
| """ The descriptor file name (with prefix) """ |
| regex = self.__class__.DESCRIPTOR_REGEX.format( |
| descriptor_type=self.descriptor_type, |
| prefix=self.prefix, |
| ) |
| desc_file = None |
| for filename in self.files: |
| if re.match(regex, filename): |
| if desc_file is not None: |
| raise PackageError("Package contains more than one descriptor") |
| desc_file = filename |
| |
| if desc_file is None: |
| raise PackageError("Could not find descriptor file in package") |
| |
| return desc_file |
| |
| @property |
| def descriptor_msg(self): |
| """ The proto-GI descriptor message """ |
| filename = self.descriptor_file |
| with self.open(filename) as hdl: |
| _, ext = os.path.splitext(filename) |
| nsd = self.serializer.from_file_hdl(hdl, ext) |
| return nsd |
| |
| @property |
| def json_descriptor(self): |
| """ The JSON serialized descriptor message""" |
| nsd = self.descriptor_msg |
| return self.serializer.to_json_string(nsd) |
| |
| @property |
| def descriptor_id(self): |
| """ The descriptor id which uniquely identifies this descriptor in the system """ |
| if not self.descriptor_msg.has_field("id"): |
| msg = "Descriptor must have an id field" |
| self._log.error(msg) |
| raise PackageError(msg) |
| |
| return self.descriptor_msg.id |
| |
| @property |
| def descriptor_name(self): |
| """ The descriptor name of this descriptor in the system """ |
| if not self.descriptor_msg.has_field("name"): |
| msg = "Descriptor name not present" |
| self._log.error(msg) |
| raise PackageError(msg) |
| |
| return self.descriptor_msg.name |
| |
| @property |
| def descriptor_version(self): |
| desc_msg = self.descriptor_msg |
| return desc_msg.version if desc_msg.has_field("version") else '' |
| |
| @property |
| def descriptor_vendor(self): |
| desc_msg = self.descriptor_msg |
| return desc_msg.vendor if desc_msg.has_field("vendor") else '' |
| |
| @classmethod |
| def get_descriptor_patterns(cls): |
| """ Returns a tuple of descriptor regex and Package Types """ |
| package_types = (VnfdPackage, NsdPackage) |
| patterns = [] |
| |
| for pkg_cls in package_types: |
| regex = cls.DESCRIPTOR_REGEX.format( |
| descriptor_type=pkg_cls.DESCRIPTOR_TYPE, |
| prefix=".*" |
| ) |
| |
| patterns.append((regex, pkg_cls)) |
| |
| return patterns |
| |
| @classmethod |
| def from_package_files(cls, log, open_fn, files): |
| """ Creates a new DescriptorPackage subclass instance from a list of files |
| |
| This classmethod detects the Package type from the package contents |
| and returns a new Package instance. |
| |
| This will NOT subsequently add the files to the package so that must |
| be done by the client |
| |
| Arguments: |
| log - A logger |
| open_fn - A function which can take a file name and mode and return |
| a file handle. |
| files - A list of files which would be added to the package after |
| intantiation |
| |
| Returns: |
| A new DescriptorPackage subclass of the correct type for the descriptor |
| |
| Raises: |
| PackageError - Package type could not be determined from the list of files. |
| """ |
| patterns = cls.get_descriptor_patterns() |
| pkg_cls = None |
| regexes = set() |
| for name in files: |
| for regex, cls in patterns: |
| regexes.add(regex) |
| if re.match(regex, name) is not None: |
| pkg_cls = cls |
| break |
| |
| if pkg_cls is None: |
| log.error("No file in archive matched known descriptor formats: %s", regexes) |
| raise PackageError("Could not determine package type from contents") |
| |
| package = pkg_cls(log, open_fn) |
| return package |
| |
| @classmethod |
| def from_descriptor_file_hdl(cls, log, file_hdl): |
| """ Creates a new DescriptorPackage from a descriptor file handle |
| |
| The descriptor file is added to the package before returning. |
| |
| Arguments: |
| log - A logger |
| file_hdl - A file handle whose name attribute can be recognized as |
| particular descriptor type. |
| |
| Returns: |
| A new DescriptorPackage subclass of the correct type for the descriptor |
| |
| Raises: |
| PackageError - Package type could not be determined from the list of files. |
| ValueError - file_hdl did not have a name attribute provided |
| """ |
| |
| package_types = (VnfdPackage, NsdPackage) |
| filename_patterns = [] |
| for package_cls in package_types: |
| filename_patterns.append( |
| (r".*{}.*".format(package_cls.DESCRIPTOR_TYPE), package_cls) |
| ) |
| |
| if not hasattr(file_hdl, 'name'): |
| raise ValueError("File descriptor must have a name attribute to create a descriptor package") |
| |
| # Iterate through the recognized patterns and assign files accordingly |
| package_cls = None |
| for pattern, cls in filename_patterns: |
| if re.match(pattern, file_hdl.name): |
| package_cls = cls |
| break |
| |
| if not package_cls: |
| raise PackageError("Could not determine package type from file name: %s" % file_hdl.name) |
| |
| _, ext = os.path.splitext(file_hdl.name) |
| try: |
| package_cls.SERIALIZER.from_file_hdl(file_hdl, ext) |
| except convert.SerializationError as e: |
| raise PackageError("Could not deserialize descriptor %s" % file_hdl.name) from e |
| |
| # Create a new file handle for each open call to prevent independent clients |
| # from affecting each other |
| file_hdl.seek(0) |
| new_hdl = io.BytesIO(file_hdl.read()) |
| |
| def do_open(file_path): |
| assert file_path == file_hdl.name |
| hdl = io.BytesIO(new_hdl.getvalue()) |
| return hdl |
| |
| desc_pkg = package_cls(log, do_open) |
| desc_pkg.add_file(file_hdl.name) |
| |
| return desc_pkg |
| |
| def get_file_mode(self, pkg_file): |
| """ Returns the file mode for the package file |
| |
| Arguments: |
| pkg_file - A file name in the package |
| |
| Returns: |
| The permission mode |
| |
| Raises: |
| PackageError - The file does not exist in the package |
| """ |
| try: |
| return self._package_file_mode_map[pkg_file] |
| except KeyError as e: |
| msg = "Could not find package_file: %s" % pkg_file |
| self._log.error(msg) |
| raise PackageError(msg) from e |
| |
| def extract_dir(self, src_dir, dest_root_dir, extract_images=False): |
| """ Extract a specific directory contents to dest_root_dir |
| |
| Arguments: |
| src_dir - A directory within the package (None means all files/directories) |
| dest_root_dir - A directory to extract directory contents to |
| extract_images - A flag indicating whether we want to extract images |
| |
| Raises: |
| ExtractError - Directory contents could not be extracted |
| """ |
| if src_dir is not None and src_dir not in self._package_dirs: |
| raise ExtractError("Could not find source dir: %s" % src_dir) |
| |
| for filename in self.files: |
| if not extract_images and image.is_image_file(filename): |
| continue |
| |
| if src_dir is not None and not filename.startswith(src_dir): |
| continue |
| |
| # Copy the contents of the file to the correct path |
| # Remove the common prefix and create the dest filename |
| if src_dir is not None: |
| fname = filename[len(src_dir):] |
| if fname[0] == '/': |
| fname = fname[1:] |
| else: |
| fname = filename |
| dest_file_path = os.path.join(dest_root_dir, fname) |
| dest_dir_path = os.path.dirname(dest_file_path) |
| if not os.path.exists(dest_dir_path): |
| os.makedirs(dest_dir_path) |
| |
| with open(dest_file_path, 'wb') as dst_hdl: |
| with self.open(filename) as src_hdl: |
| shutil.copyfileobj(src_hdl, dst_hdl, 10 * 1024 * 1024) |
| |
| # Set the file mode to original |
| os.chmod(dest_file_path, self._package_file_mode_map[filename]) |
| |
| def insert_file(self, new_file, dest_file, rel_path, mode=0o777): |
| self.add_file(rel_path, mode) |
| |
| try: |
| # Copy the contents of the file to the correct path |
| # For folder creation (or nested folders), dest_file appears w/ trailing "/" like: dir1/ or dir1/dir2/ |
| # For regular file upload, dest_file appears as dir1/abc.txt |
| |
| dest_dir_path = os.path.dirname(dest_file) |
| if not os.path.isdir(dest_dir_path): |
| os.makedirs(dest_dir_path) |
| if not os.path.basename(dest_file): |
| self._log.debug("Created dir path, no filename to insert in {}, skipping..".format(dest_dir_path)) |
| return |
| |
| with open(dest_file, 'wb') as dst_hdl: |
| with open(new_file, 'rb') as src_hdl: |
| shutil.copyfileobj(src_hdl, dst_hdl, 10 * 1024 * 1024) |
| |
| # Set the file mode to original |
| os.chmod(dest_file, self._package_file_mode_map[rel_path]) |
| except Exception as e: |
| # Clear the file when an exception happens |
| if os.path.isfile(dest_file): |
| os.remove(dest_file) |
| |
| raise PackageAppendError(str(e)) |
| |
| def delete_file(self, dest_file, rel_path): |
| self.remove_file(rel_path) |
| |
| try: |
| os.remove(dest_file) |
| except Exception as e: |
| raise PackageAppendError(str(e)) |
| |
| def extract_file(self, src_file, dest_file): |
| """ Extract a specific package file to dest_file |
| |
| The destination directory will be created if it does not exist. |
| |
| Arguments: |
| src_file - A file within the package |
| dest_file - A file path to extract file contents to |
| |
| Raises: |
| ExtractError - Directory contents could not be extracted |
| """ |
| if src_file not in self._package_file_mode_map: |
| msg = "Could not find source file %s" % src_file |
| self._log.error(msg) |
| raise ExtractError(msg) |
| |
| # Copy the contents of the file to the correct path |
| dest_dir_path = os.path.dirname(dest_file) |
| if not os.path.isdir(dest_dir_path): |
| os.makedirs(dest_dir_path) |
| |
| with open(dest_file, 'wb') as dst_hdl: |
| with self.open(src_file) as src_hdl: |
| shutil.copyfileobj(src_hdl, dst_hdl, 10 * 1024 * 1024) |
| |
| # Set the file mode to original |
| os.chmod(dest_file, self._package_file_mode_map[src_file]) |
| |
| def extract(self, dest_root_dir, extract_images=False): |
| """ Extract all package contents to a destination directory |
| |
| Arguments: |
| dest_root_dir - The directory to extract package contents to |
| |
| Raises: |
| NotADirectoryError - dest_root_dir is not a directory |
| """ |
| def find_prefix(): |
| """ Find comon prefix of all files in package. This prefix will be |
| used to collapse directory structure during extraction to eliminate |
| empty nested folders. |
| """ |
| common_dir = set() |
| for f in self.files: |
| common_dir.add(os.path.dirname(f)) |
| prefix = os.path.commonprefix(list(common_dir)) |
| return prefix if prefix else None |
| |
| if not os.path.isdir(dest_root_dir): |
| raise NotADirectoryError(dest_root_dir) |
| |
| self.extract_dir(find_prefix(), dest_root_dir, extract_images) |
| |
| def open(self, rel_path): |
| """ Open a file contained in the package in read-only, binary mode. |
| |
| Arguments: |
| rel_path - The file path within the package |
| |
| Returns: |
| A file-like object opened in read-only mode. |
| |
| Raises: |
| PackageError - The file could not be opened |
| """ |
| try: |
| return self._open_fn(rel_path) |
| except Exception as e: |
| msg = "Could not open file from package: %s" % rel_path |
| self._log.warning(msg) |
| raise PackageError(msg) from e |
| |
| def add_file(self, rel_path, mode=0o777): |
| """ Add a file to the package. |
| |
| The file should be specified as a relative path to the package |
| root. The open_fn provided in the constructor must be able to |
| take the relative path and open the actual source file from |
| wherever the file actually is stored. |
| |
| If the file's parent directories do not yet exist, add them to |
| the package. |
| |
| Arguments: |
| rel_path - The file path relative to the top of the package. |
| mode - The permission mode the file should be stored with so |
| it can be extracted with the correct permissions. |
| |
| Raises: |
| PackageError - The file could not be added to the package |
| """ |
| if not rel_path: |
| raise PackageError("Empty file name added") |
| |
| if rel_path in self._package_file_mode_map: |
| raise PackageError("File %s already exists in package" % rel_path) |
| |
| # If the file's directory is not in the package add it. |
| rel_dir = os.path.dirname(rel_path) |
| while rel_dir: |
| self._package_dirs.add(rel_dir) |
| rel_dir = os.path.dirname(rel_dir) |
| |
| self._package_file_mode_map[rel_path] = mode |
| |
| def remove_file(self, rel_path): |
| if not rel_path: |
| raise PackageError("Empty file name added") |
| |
| if rel_path not in self._package_file_mode_map: |
| raise PackageError("File %s does not exist in package" % rel_path) |
| |
| del self._package_file_mode_map[rel_path] |
| |
| def add_dir(self, rel_path): |
| """ Add a directory to the package |
| |
| Arguments: |
| rel_path - The directories relative path. |
| |
| Raises: |
| PackageError - A file already exists in the package with the same name. |
| """ |
| if rel_path in self._package_file_mode_map: |
| raise PackageError("File already exists with the same name: %s", rel_path) |
| |
| if rel_path in self._package_dirs: |
| self._log.warning("%s directory already exists", rel_path) |
| return |
| |
| self._package_dirs.add(rel_path) |
| |
| |
| class NsdPackage(DescriptorPackage): |
| DESCRIPTOR_TYPE = "nsd" |
| SERIALIZER = convert.RwNsdSerializer() |
| |
| @property |
| def descriptor_type(self): |
| return "nsd" |
| |
| @property |
| def serializer(self): |
| return NsdPackage.SERIALIZER |
| |
| |
| class VnfdPackage(DescriptorPackage): |
| DESCRIPTOR_TYPE = "vnfd" |
| SERIALIZER = convert.RwVnfdSerializer() |
| |
| @property |
| def descriptor_type(self): |
| return "vnfd" |
| |
| @property |
| def serializer(self): |
| return VnfdPackage.SERIALIZER |
| |
| class PackageConstructValidator(object): |
| |
| def __init__(self, log): |
| self._log = log |
| |
| def validate(self, package): |
| """ Validate presence of descriptor file (.yaml) at the top level in the |
| package folder structure. |
| |
| Arguments: |
| package - The Descriptor Package being validated. |
| Returns: |
| None |
| Raises: |
| PackageValidationError - The package validation failed for some |
| generic reason. |
| """ |
| pass |
| desc_file = package.descriptor_file |
| prefix, desc_file = package.prefix.rstrip('/'), desc_file.rstrip('/') |
| |
| if os.path.dirname(desc_file) != prefix: |
| msg = "Descriptor file {} not found in expcted location {}".format(desc_file, prefix) |
| self._log.error(msg) |
| raise PackageValidationError(msg) |
| |
| |
| class PackageChecksumValidator(object): |
| """ This class uses the checksums.txt file in the package |
| and validates that all files in the package match the checksum that exists within |
| the file. |
| """ |
| CHECKSUM_FILE = "{prefix}checksums.txt" |
| |
| def __init__(self, log): |
| self._log = log |
| self.validated_file_checksums = {} |
| |
| @classmethod |
| def get_package_checksum_file(cls, package): |
| checksum_file = cls.CHECKSUM_FILE.format(prefix=package.prefix) |
| if checksum_file not in package.files: |
| raise FileNotFoundError("%s does not exist in archive" % checksum_file) |
| |
| return checksum_file |
| |
| @property |
| def checksums(self): |
| return self.validated_file_checksums |
| |
| def validate(self, package): |
| """ Validate file checksums match that in the checksums.txt |
| |
| Arguments: |
| package - The Descriptor Package which possiblity contains checksums.txt |
| |
| Returns: A dictionary of files that were validated by the checksums.txt |
| along with their checksums |
| |
| Raises: |
| PackageValidationError - The package validation failed for some |
| generic reason. |
| PackageFileChecksumError - A file within the package did not match the |
| checksum within checksums.txt |
| """ |
| |
| try: |
| checksum_file = PackageChecksumValidator.get_package_checksum_file(package) |
| with package.open(checksum_file) as checksum_hdl: |
| archive_checksums = checksums.ArchiveChecksums.from_file_desc(checksum_hdl) |
| except (FileNotFoundError, PackageError) as e: |
| self._log.warning("Could not open package checksum file. Not validating checksums.") |
| return self.validated_file_checksums |
| |
| for pkg_file in package.files: |
| if pkg_file == checksum_file: |
| continue |
| |
| pkg_file_no_prefix = pkg_file.replace(package.prefix, "", 1) |
| if pkg_file_no_prefix not in archive_checksums: |
| self._log.warning("File %s not found in checksum file %s", |
| pkg_file, checksum_file) |
| continue |
| |
| try: |
| with package.open(pkg_file) as pkg_file_hdl: |
| file_checksum = checksums.checksum(pkg_file_hdl) |
| except PackageError as e: |
| msg = "Could not read package file {} for checksum validation: {}".format( |
| pkg_file, str(e)) |
| self._log.error(msg) |
| raise PackageValidationError(msg) from e |
| |
| if archive_checksums[pkg_file_no_prefix] != file_checksum: |
| msg = "{} checksum ({}) did not match expected checksum ({})".format( |
| pkg_file, file_checksum, archive_checksums[pkg_file_no_prefix] |
| ) |
| self._log.error(msg) |
| raise PackageFileChecksumError(pkg_file) |
| |
| self.validated_file_checksums[pkg_file] = file_checksum |
| |
| |
| class TarPackageArchive(object): |
| """ This class represents a package stored within a tar.gz archive file """ |
| def __init__(self, log, tar_file_hdl, mode="r"): |
| self._log = log |
| self._tar_filepath = tar_file_hdl |
| self._tar_infos = {} |
| |
| self._tarfile = tarfile.open(fileobj=tar_file_hdl, mode=mode) |
| |
| self._load_archive() |
| |
| def __repr__(self): |
| return "TarPackageArchive(%s)" % self._tar_filepath |
| |
| def _get_members(self): |
| return [info for info in self._tarfile.getmembers()] |
| |
| def _load_archive(self): |
| self._tar_infos = {info.name: info for info in self._get_members() if info.name} |
| |
| def __del__(self): |
| self.close() |
| |
| def close(self): |
| """ Close the opened tarfile""" |
| if self._tarfile is not None: |
| self._tarfile.close() |
| self._tarfile = None |
| |
| @property |
| def filenames(self): |
| """ The list of file members within the tar file """ |
| return [name for name in self._tar_infos if tarfile.TarInfo.isfile(self._tar_infos[name])] |
| |
| def open_file(self, rel_file_path): |
| """ Opens a file within the archive as read-only, byte mode. |
| |
| Arguments: |
| rel_file_path - The file path within the archive to open |
| |
| Returns: |
| A file like object (see tarfile.extractfile()) |
| |
| Raises: |
| ArchiveError - The file could not be opened for some generic reason. |
| """ |
| if rel_file_path not in self._tar_infos: |
| raise ArchiveError("Could not find %s in tar file", rel_file_path) |
| |
| try: |
| return self._tarfile.extractfile(rel_file_path) |
| except tarfile.TarError as e: |
| msg = "Failed to read file {} from tarfile {}: {}".format( |
| rel_file_path, self._tar_filepath, str(e) |
| ) |
| self._log.error(msg) |
| raise ArchiveError(msg) from e |
| |
| def create_package(self): |
| """ Creates a Descriptor package from the archive contents |
| |
| Returns: |
| A DescriptorPackage of the correct descriptor type |
| """ |
| package = DescriptorPackage.from_package_files(self._log, self.open_file, self.filenames) |
| for pkg_file in self.filenames: |
| package.add_file(pkg_file, self._tar_infos[pkg_file].mode) |
| |
| return package |
| |
| |
| class TemporaryPackage(object): |
| """ This class is a container for a temporary file-backed package |
| |
| This class contains a DescriptorPackage and can be used in place of one. |
| Provides a useful context manager which will close and destroy the file |
| that is backing the DescriptorPackage on exit. |
| """ |
| def __init__(self, log, package, file_hdl): |
| self._log = log |
| self._package = package |
| self._file_hdl = file_hdl |
| |
| if not hasattr(self._file_hdl, "name"): |
| raise ValueError("File handle must have a name attribute") |
| |
| def __getattr__(self, attr): |
| return getattr(self._package, attr) |
| |
| def __enter__(self): |
| return self._package |
| |
| def __exit__(self, type, value, tb): |
| self.close() |
| |
| def filename(self): |
| """ Returns the filepath with is backing the Package """ |
| return self._file_hdl.name |
| |
| def package(self): |
| """ The contained DescriptorPackage instance """ |
| return self._package |
| |
| def close(self): |
| """ Close and remove the backed file """ |
| filename = self._file_hdl.name |
| |
| try: |
| self._file_hdl.close() |
| except OSError as e: |
| self._log.warning("Failed to close package file: %s", str(e)) |
| |
| try: |
| os.remove(filename) |
| except OSError as e: |
| self._log.warning("Failed to remove package file: %s", str(e)) |