rwlaunchpad/plugins/rwlaunchpadtasklet/rift/package/package.py

   1
   2 #
   3 #   Copyright 2016 RIFT.IO Inc
   4 #
   5 #   Licensed under the Apache License, Version 2.0 (the "License");
   6 #   you may not use this file except in compliance with the License.
   7 #   You may obtain a copy of the License at
   8 #
   9 #       http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 #   Unless required by applicable law or agreed to in writing, software
  12 #   distributed under the License is distributed on an "AS IS" BASIS,
  13 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 #   See the License for the specific language governing permissions and
  15 #   limitations under the License.
  16 #
  17
  18 import io
  19 import os
  20 import re
  21 import shutil
  22 import tarfile
  23
  24 from . import checksums
  25 from . import convert
  26 from . import image
  27
  28
  29 class ArchiveError(Exception):
  30     pass
  31
  32
  33 class ExtractError(Exception):
  34     pass
  35
  36
  37 class PackageError(Exception):
  38     pass
  39
  40
  41 class PackageValidationError(Exception):
  42     pass
  43
  44
  45 class PackageFileChecksumError(PackageValidationError):
  46     def __init__(self, filename):
  47         self.filename = filename
  48         super().__init__("Checksum mismatch for {}".format(filename))
  49
  50
  51 class DescriptorPackage(object):
  52     """ This class provides an base class for a descriptor package representing
  53
  54     A descriptor package is a package which contains a single descriptor and any
  55     associated files (logos, charms, scripts, etc).  This package representation
  56     attempts to be agnostic as to where the package files are being stored
  57     (in memory, on disk, etc).
  58
  59     The package provides a simple interface to interact with the files within the
  60     package and access the contained descriptor.
  61     """
  62     DESCRIPTOR_REGEX = r"{prefix}({descriptor_type}/[^/]*|[^/]*{descriptor_type})\.(xml|yml|yaml|json)$"
  63
  64     def __init__(self, log, open_fn):
  65         self._log = log
  66         self._open_fn = open_fn
  67
  68         self._package_file_mode_map = {}
  69         self._package_dirs = set()
  70
  71     @property
  72     def prefix(self):
  73         """ Return the leading parent directories shared by all files in the package
  74
  75         In order to remain flexible as to where tar was invoked to create the package,
  76         the prefix represents the common parent directory path which all files in the
  77         package have in common.
  78         """
  79         entries = list(self._package_file_mode_map) + list(self._package_dirs)
  80
  81         if len(entries) > 1:
  82             prefix = os.path.commonprefix(entries)
  83             if prefix and not prefix.endswith("/"):
  84                 prefix += "/"
  85         elif len(entries) == 1:
  86             entry = entries[0]
  87             if "/" in entry:
  88                 prefix = os.path.dirname(entry) + "/"
  89             else:
  90                 prefix = ""
  91         else:
  92             prefix = ""
  93
  94         return prefix
  95
  96     @property
  97     def files(self):
  98         """ Return all files (with the prefix) in the package """
  99         return list(self._package_file_mode_map)
 100
 101     @property
 102     def dirs(self):
 103         """ Return all directories in the package """
 104         return list(self._package_dirs)
 105
 106     @property
 107     def descriptor_type(self):
 108         """ A shorthand name for the type of descriptor (e.g. nsd)"""
 109         raise NotImplementedError("Subclass must implement this property")
 110
 111     @property
 112     def serializer(self):
 113         """ An instance of convert.ProtoMessageSerializer """
 114         raise NotImplementedError("Subclass must implement this property")
 115
 116     @property
 117     def descriptor_file(self):
 118         """ The descriptor file name (with prefix) """
 119         regex = self.__class__.DESCRIPTOR_REGEX.format(
 120                 descriptor_type=self.descriptor_type,
 121                 prefix=self.prefix,
 122                 )
 123         desc_file = None
 124         for filename in self.files:
 125             if re.match(regex, filename):
 126                 if desc_file is not None:
 127                     raise PackageError("Package contains more than one descriptor")
 128                 desc_file = filename
 129
 130         if desc_file is None:
 131             raise PackageError("Could not find descriptor file in package")
 132
 133         return desc_file
 134
 135     @property
 136     def descriptor_msg(self):
 137         """ The proto-GI descriptor message """
 138         filename = self.descriptor_file
 139         with self.open(filename) as hdl:
 140             _, ext = os.path.splitext(filename)
 141             nsd = self.serializer.from_file_hdl(hdl, ext)
 142             return nsd
 143
 144     @property
 145     def json_descriptor(self):
 146         """  The JSON serialized descriptor message"""
 147         nsd = self.descriptor_msg
 148         return self.serializer.to_json_string(nsd)
 149
 150     @property
 151     def descriptor_id(self):
 152         """  The descriptor id which uniquely identifies this descriptor in the system """
 153         if not self.descriptor_msg.has_field("id"):
 154             msg = "Descriptor must have an id field"
 155             self._log.error(msg)
 156             raise PackageError(msg)
 157
 158         return self.descriptor_msg.id
 159
 160     @classmethod
 161     def get_descriptor_patterns(cls):
 162         """ Returns a tuple of descriptor regex and Package Types  """
 163         package_types = (VnfdPackage, NsdPackage)
 164         patterns = []
 165
 166         for pkg_cls in package_types:
 167             regex = cls.DESCRIPTOR_REGEX.format(
 168                     descriptor_type=pkg_cls.DESCRIPTOR_TYPE,
 169                     prefix=".*"
 170                     )
 171
 172             patterns.append((regex, pkg_cls))
 173
 174         return patterns
 175
 176     @classmethod
 177     def from_package_files(cls, log, open_fn, files):
 178         """ Creates a new DescriptorPackage subclass instance from a list of files
 179
 180         This classmethod detects the Package type from the package contents
 181         and returns a new Package instance.
 182
 183         This will NOT subsequently add the files to the package so that must
 184         be done by the client
 185
 186         Arguments:
 187             log - A logger
 188             open_fn - A function which can take a file name and mode and return
 189                       a file handle.
 190             files - A list of files which would be added to the package after
 191                     intantiation
 192
 193         Returns:
 194             A new DescriptorPackage subclass of the correct type for the descriptor
 195
 196         Raises:
 197             PackageError - Package type could not be determined from the list of files.
 198         """
 199         patterns = cls.get_descriptor_patterns()
 200         pkg_cls = None
 201         regexes = set()
 202         for name in files:
 203             for regex, cls in patterns:
 204                 regexes.add(regex)
 205                 if re.match(regex, name) is not None:
 206                     pkg_cls = cls
 207                     break
 208
 209         if pkg_cls is None:
 210             log.error("No file in archive matched known descriptor formats: %s", regexes)
 211             raise PackageError("Could not determine package type from contents")
 212
 213         package = pkg_cls(log, open_fn)
 214         return package
 215
 216     @classmethod
 217     def from_descriptor_file_hdl(cls, log, file_hdl):
 218         """ Creates a new DescriptorPackage from a descriptor file handle
 219
 220         The descriptor file is added to the package before returning.
 221
 222         Arguments:
 223             log - A logger
 224             file_hdl - A file handle whose name attribute can be recognized as
 225                        particular descriptor type.
 226
 227         Returns:
 228             A new DescriptorPackage subclass of the correct type for the descriptor
 229
 230         Raises:
 231             PackageError - Package type could not be determined from the list of files.
 232             ValueError - file_hdl did not have a name attribute provided
 233         """
 234
 235         package_types = (VnfdPackage, NsdPackage)
 236         filename_patterns = []
 237         for package_cls in package_types:
 238             filename_patterns.append(
 239                     (r".*{}.*".format(package_cls.DESCRIPTOR_TYPE), package_cls)
 240                     )
 241
 242         if not hasattr(file_hdl, 'name'):
 243             raise ValueError("File descriptor must have a name attribute to create a descriptor package")
 244
 245         # Iterate through the recognized patterns and assign files accordingly
 246         package_cls = None
 247         for pattern, cls in filename_patterns:
 248             if re.match(pattern, file_hdl.name):
 249                 package_cls = cls
 250                 break
 251
 252         if not package_cls:
 253             raise PackageError("Could not determine package type from file name: %s" % file_hdl.name)
 254
 255         _, ext = os.path.splitext(file_hdl.name)
 256         try:
 257             package_cls.SERIALIZER.from_file_hdl(file_hdl, ext)
 258         except convert.SerializationError as e:
 259             raise PackageError("Could not deserialize descriptor %s" % file_hdl.name) from e
 260
 261         # Create a new file handle for each open call to prevent independent clients
 262         # from affecting each other
 263         file_hdl.seek(0)
 264         new_hdl = io.BytesIO(file_hdl.read())
 265
 266         def do_open(file_path):
 267             assert file_path == file_hdl.name
 268             hdl = io.BytesIO(new_hdl.getvalue())
 269             return hdl
 270
 271         desc_pkg = package_cls(log, do_open)
 272         desc_pkg.add_file(file_hdl.name)
 273
 274         return desc_pkg
 275
 276     def get_file_mode(self, pkg_file):
 277         """ Returns the file mode for the package file
 278
 279         Arguments:
 280             pkg_file - A file name in the package
 281
 282         Returns:
 283             The permission mode
 284
 285         Raises:
 286             PackageError - The file does not exist in the package
 287         """
 288         try:
 289             return self._package_file_mode_map[pkg_file]
 290         except KeyError as e:
 291             msg = "Could not find package_file: %s" % pkg_file
 292             self._log.error(msg)
 293             raise PackageError(msg) from e
 294
 295     def extract_dir(self, src_dir, dest_root_dir, extract_images=False):
 296         """ Extract a specific directory contents to dest_root_dir
 297
 298         Arguments:
 299             src_dir - A directory within the package (None means all files/directories)
 300             dest_root_dir - A directory to extract directory contents to
 301             extract_images - A flag indicating whether we want to extract images
 302
 303         Raises:
 304             ExtractError - Directory contents could not be extracted
 305         """
 306         if src_dir is not None and src_dir not in self._package_dirs:
 307             raise ExtractError("Could not find source dir: %s" % src_dir)
 308
 309         for filename in self.files:
 310             if not extract_images and image.is_image_file(filename):
 311                 continue
 312
 313             if src_dir is not None and not filename.startswith(src_dir):
 314                 continue
 315
 316             # Copy the contents of the file to the correct path
 317             # Remove the common prefix and create the dest filename
 318             if src_dir is not None:
 319                 fname = filename[len(src_dir):]
 320                 if fname[0] == '/':
 321                     fname = fname[1:]
 322             else:
 323                 fname = filename
 324             dest_file_path = os.path.join(dest_root_dir, fname)
 325             dest_dir_path = os.path.dirname(dest_file_path)
 326             if not os.path.exists(dest_dir_path):
 327                 os.makedirs(dest_dir_path)
 328
 329             with open(dest_file_path, 'wb') as dst_hdl:
 330                 with self.open(filename) as src_hdl:
 331                     shutil.copyfileobj(src_hdl, dst_hdl, 10 * 1024 * 1024)
 332
 333                     # Set the file mode to original
 334                     os.chmod(dest_file_path, self._package_file_mode_map[filename])
 335
 336     def extract_file(self, src_file, dest_file):
 337         """ Extract a specific package file to dest_file
 338
 339         The destination directory will be created if it does not exist.
 340
 341         Arguments:
 342             src_file - A file within the package
 343             dest_file - A file path to extract file contents to
 344
 345         Raises:
 346             ExtractError - Directory contents could not be extracted
 347         """
 348         if src_file not in self._package_file_mode_map:
 349             msg = "Could not find source file %s" % src_file
 350             self._log.error(msg)
 351             raise ExtractError(msg)
 352
 353         # Copy the contents of the file to the correct path
 354         dest_dir_path = os.path.dirname(dest_file)
 355         if not os.path.isdir(dest_dir_path):
 356             os.makedirs(dest_dir_path)
 357
 358         with open(dest_file, 'wb') as dst_hdl:
 359             with self.open(src_file) as src_hdl:
 360                 shutil.copyfileobj(src_hdl, dst_hdl, 10 * 1024 * 1024)
 361
 362                 # Set the file mode to original
 363                 os.chmod(dest_file, self._package_file_mode_map[src_file])
 364
 365     def extract(self, dest_root_dir, extract_images=False):
 366         """ Extract all package contents to a destination directory
 367
 368         Arguments:
 369             dest_root_dir - The directory to extract package contents to
 370
 371         Raises:
 372             NotADirectoryError - dest_root_dir is not a directory
 373         """
 374         if not os.path.isdir(dest_root_dir):
 375             raise NotADirectoryError(dest_root_dir)
 376
 377         self.extract_dir(None, dest_root_dir, extract_images)
 378
 379     def open(self, rel_path):
 380         """ Open a file contained in the package in read-only, binary mode.
 381
 382         Arguments:
 383             rel_path - The file path within the package
 384
 385         Returns:
 386             A file-like object opened in read-only mode.
 387
 388         Raises:
 389             PackageError - The file could not be opened
 390         """
 391         try:
 392             return self._open_fn(rel_path)
 393         except Exception as e:
 394             msg = "Could not open file from package: %s" % rel_path
 395             self._log.warning(msg)
 396             raise PackageError(msg) from e
 397
 398     def add_file(self, rel_path, mode=0o777):
 399         """ Add a file to the package.
 400
 401         The file should be specified as a relative path to the package
 402         root.  The open_fn provided in the constructor must be able to
 403         take the relative path and open the actual source file from
 404         wherever the file actually is stored.
 405
 406         If the file's parent directories do not yet exist, add them to
 407         the package.
 408
 409         Arguments:
 410             rel_path - The file path relative to the top of the package.
 411             mode - The permission mode the file should be stored with so
 412                    it can be extracted with the correct permissions.
 413
 414         Raises:
 415             PackageError - The file could not be added to the package
 416         """
 417         if not rel_path:
 418             raise PackageError("Empty file name added")
 419
 420         if rel_path in self._package_file_mode_map:
 421             raise PackageError("File %s already exists in package" % rel_path)
 422
 423         # If the file's directory is not in the package add it.
 424         rel_dir = os.path.dirname(rel_path)
 425         while rel_dir:
 426             self._package_dirs.add(rel_dir)
 427             rel_dir = os.path.dirname(rel_dir)
 428
 429         self._package_file_mode_map[rel_path] = mode
 430
 431     def add_dir(self, rel_path):
 432         """ Add a directory to the package
 433
 434         Arguments:
 435             rel_path - The directories relative path.
 436
 437         Raises:
 438             PackageError - A file already exists in the package with the same name.
 439         """
 440         if rel_path in self._package_file_mode_map:
 441             raise PackageError("File already exists with the same name: %s", rel_path)
 442
 443         if rel_path in self._package_dirs:
 444             self._log.warning("%s directory already exists", rel_path)
 445             return
 446
 447         self._package_dirs.add(rel_path)
 448
 449
 450 class NsdPackage(DescriptorPackage):
 451     DESCRIPTOR_TYPE = "nsd"
 452     SERIALIZER = convert.RwNsdSerializer()
 453
 454     @property
 455     def descriptor_type(self):
 456         return "nsd"
 457
 458     @property
 459     def serializer(self):
 460         return NsdPackage.SERIALIZER
 461
 462
 463 class VnfdPackage(DescriptorPackage):
 464     DESCRIPTOR_TYPE = "vnfd"
 465     SERIALIZER = convert.RwVnfdSerializer()
 466
 467     @property
 468     def descriptor_type(self):
 469         return "vnfd"
 470
 471     @property
 472     def serializer(self):
 473         return VnfdPackage.SERIALIZER
 474
 475
 476 class PackageChecksumValidator(object):
 477     """  This class uses the checksums.txt file in the package
 478     and validates that all files in the package match the checksum that exists within
 479     the file.
 480     """
 481     CHECKSUM_FILE = "{prefix}checksums.txt"
 482
 483     def __init__(self, log):
 484         self._log = log
 485
 486     @classmethod
 487     def get_package_checksum_file(cls, package):
 488         checksum_file = cls.CHECKSUM_FILE.format(prefix=package.prefix)
 489         if checksum_file not in package.files:
 490             raise FileNotFoundError("%s does not exist in archive" % checksum_file)
 491
 492         return checksum_file
 493
 494     def validate(self, package):
 495         """ Validate file checksums match that in the checksums.txt
 496
 497         Arguments:
 498             package - The Descriptor Package which possiblity contains checksums.txt
 499
 500         Returns: A dictionary of files that were validated by the checksums.txt
 501                  along with their checksums
 502
 503         Raises:
 504             PackageValidationError - The package validation failed for some
 505               generic reason.
 506             PackageFileChecksumError - A file within the package did not match the
 507               checksum within checksums.txt
 508         """
 509         validated_file_checksums = {}
 510
 511         try:
 512             checksum_file = PackageChecksumValidator.get_package_checksum_file(package)
 513             with package.open(checksum_file) as checksum_hdl:
 514                 archive_checksums = checksums.ArchiveChecksums.from_file_desc(checksum_hdl)
 515         except (FileNotFoundError, PackageError) as e:
 516             self._log.warning("Could not open package checksum file.  Not validating checksums.")
 517             return validated_file_checksums
 518
 519         for pkg_file in package.files:
 520             if pkg_file == checksum_file:
 521                 continue
 522
 523             pkg_file_no_prefix = pkg_file.replace(package.prefix, "", 1)
 524             if pkg_file_no_prefix not in archive_checksums:
 525                 self._log.warning("File %s not found in checksum file %s",
 526                                   pkg_file, checksum_file)
 527                 continue
 528
 529             try:
 530                 with package.open(pkg_file) as pkg_file_hdl:
 531                     file_checksum = checksums.checksum(pkg_file_hdl)
 532             except PackageError as e:
 533                 msg = "Could not read package file {} for checksum validation: {}".format(
 534                       pkg_file, str(e))
 535                 self._log.error(msg)
 536                 raise PackageValidationError(msg) from e
 537
 538             if archive_checksums[pkg_file_no_prefix] != file_checksum:
 539                 msg = "{} checksum ({}) did match expected checksum ({})".format(
 540                         pkg_file, file_checksum, archive_checksums[pkg_file_no_prefix]
 541                         )
 542                 self._log.error(msg)
 543                 raise PackageFileChecksumError(pkg_file)
 544
 545             validated_file_checksums[pkg_file] = file_checksum
 546
 547         return validated_file_checksums
 548
 549
 550 class TarPackageArchive(object):
 551     """  This class represents a package stored within a tar.gz archive file """
 552     def __init__(self, log, tar_file_hdl, mode="r"):
 553         self._log = log
 554         self._tar_filepath = tar_file_hdl
 555         self._tar_infos = {}
 556
 557         self._tarfile = tarfile.open(fileobj=tar_file_hdl, mode=mode)
 558
 559         self._load_archive()
 560
 561     def __repr__(self):
 562         return "TarPackageArchive(%s)" % self._tar_filepath
 563
 564     def _get_members(self):
 565         return [info for info in self._tarfile.getmembers()]
 566
 567     def _load_archive(self):
 568         self._tar_infos = {info.name: info for info in self._get_members() if info.name}
 569
 570     def __del__(self):
 571         self.close()
 572
 573     def close(self):
 574         """ Close the opened tarfile"""
 575         if self._tarfile is not None:
 576             self._tarfile.close()
 577             self._tarfile = None
 578
 579     @property
 580     def filenames(self):
 581         """ The list of file members within the tar file """
 582         return [name for name in self._tar_infos if tarfile.TarInfo.isfile(self._tar_infos[name])]
 583
 584     def open_file(self, rel_file_path):
 585         """ Opens a file within the archive as read-only, byte mode.
 586
 587         Arguments:
 588             rel_file_path - The file path within the archive to open
 589
 590         Returns:
 591             A file like object (see tarfile.extractfile())
 592
 593         Raises:
 594             ArchiveError - The file could not be opened for some generic reason.
 595         """
 596         if rel_file_path not in self._tar_infos:
 597             raise ArchiveError("Could not find %s in tar file", rel_file_path)
 598
 599         try:
 600             return self._tarfile.extractfile(rel_file_path)
 601         except tarfile.TarError as e:
 602             msg = "Failed to read file {} from tarfile {}: {}".format(
 603                   rel_file_path, self._tar_filepath, str(e)
 604                   )
 605             self._log.error(msg)
 606             raise ArchiveError(msg) from e
 607
 608     def create_package(self):
 609         """  Creates a Descriptor package from the archive contents
 610
 611         Returns:
 612             A DescriptorPackage of the correct descriptor type
 613         """
 614         package = DescriptorPackage.from_package_files(self._log, self.open_file, self.filenames)
 615         for pkg_file in self.filenames:
 616             package.add_file(pkg_file, self._tar_infos[pkg_file].mode)
 617
 618         return package
 619
 620
 621 class TemporaryPackage(object):
 622     """  This class is a container for a temporary file-backed package
 623
 624     This class contains a DescriptorPackage and can be used in place of one.
 625     Provides a useful context manager which will close and destroy the file
 626     that is backing the DescriptorPackage on exit.
 627     """
 628     def __init__(self, log, package, file_hdl):
 629         self._log = log
 630         self._package = package
 631         self._file_hdl = file_hdl
 632
 633         if not hasattr(self._file_hdl, "name"):
 634             raise ValueError("File handle must have a name attribute")
 635
 636     def __getattr__(self, attr):
 637         return getattr(self._package, attr)
 638
 639     def __enter__(self):
 640         return self._package
 641
 642     def __exit__(self, type, value, tb):
 643         self.close()
 644
 645     def filename(self):
 646         """ Returns the filepath with is backing the Package """
 647         return self._file_hdl.name
 648
 649     def package(self):
 650         """ The contained DescriptorPackage instance """
 651         return self._package
 652
 653     def close(self):
 654         """ Close and remove the backed file """
 655         filename = self._file_hdl.name
 656
 657         try:
 658             self._file_hdl.close()
 659         except OSError as e:
 660             self._log.warning("Failed to close package file: %s", str(e))
 661
 662         try:
 663             os.remove(filename)
 664         except OSError as e:
 665             self._log.warning("Failed to remove package file: %s", str(e))