[RIFT 16413, 16414] Unittest failures fixed for utest_package, utest_publisher_dts
[osm/SO.git] / rwlaunchpad / plugins / rwlaunchpadtasklet / rift / package / package.py
1
2 #
3 # Copyright 2016 RIFT.IO Inc
4 #
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
8 #
9 # http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16 #
17
18 import io
19 import os
20 import re
21 import shutil
22 import tarfile
23
24 from . import checksums
25 from . import convert
26 from . import image
27
28
29 class ArchiveError(Exception):
30 pass
31
32
33 class ExtractError(Exception):
34 pass
35
36
37 class PackageError(Exception):
38 pass
39
40
41 class PackageValidationError(Exception):
42 pass
43
44
45 class PackageAppendError(Exception):
46 pass
47
48
49 class PackageFileChecksumError(PackageValidationError):
50 def __init__(self, filename):
51 self.filename = filename
52 super().__init__("Checksum mismatch for {}".format(filename))
53
54
55 class DescriptorPackage(object):
56 """ This class provides an base class for a descriptor package representing
57
58 A descriptor package is a package which contains a single descriptor and any
59 associated files (logos, charms, scripts, etc). This package representation
60 attempts to be agnostic as to where the package files are being stored
61 (in memory, on disk, etc).
62
63 The package provides a simple interface to interact with the files within the
64 package and access the contained descriptor.
65 """
66 DESCRIPTOR_REGEX = r"{prefix}({descriptor_type}/[^/]*|[^/]*{descriptor_type})\.(xml|yml|yaml|json)$"
67
68 def __init__(self, log, open_fn):
69 self._log = log
70 self._open_fn = open_fn
71
72 self._package_file_mode_map = {}
73 self._package_dirs = set()
74
75 @property
76 def prefix(self):
77 """ Return the leading parent directories shared by all files in the package
78
79 In order to remain flexible as to where tar was invoked to create the package,
80 the prefix represents the common parent directory path which all files in the
81 package have in common.
82 """
83 entries = list(self._package_file_mode_map) + list(self._package_dirs)
84
85 if len(entries) > 1:
86 prefix = os.path.commonprefix(entries)
87 if prefix and not prefix.endswith("/"):
88 prefix += "/"
89 elif len(entries) == 1:
90 entry = entries[0]
91 if "/" in entry:
92 prefix = os.path.dirname(entry) + "/"
93 else:
94 prefix = ""
95 else:
96 prefix = ""
97
98 return prefix
99
100 @property
101 def files(self):
102 """ Return all files (with the prefix) in the package """
103 return list(self._package_file_mode_map)
104
105 @property
106 def dirs(self):
107 """ Return all directories in the package """
108 return list(self._package_dirs)
109
110 @property
111 def descriptor_type(self):
112 """ A shorthand name for the type of descriptor (e.g. nsd)"""
113 raise NotImplementedError("Subclass must implement this property")
114
115 @property
116 def serializer(self):
117 """ An instance of convert.ProtoMessageSerializer """
118 raise NotImplementedError("Subclass must implement this property")
119
120 @property
121 def descriptor_file(self):
122 """ The descriptor file name (with prefix) """
123 regex = self.__class__.DESCRIPTOR_REGEX.format(
124 descriptor_type=self.descriptor_type,
125 prefix=self.prefix,
126 )
127 desc_file = None
128 for filename in self.files:
129 if re.match(regex, filename):
130 if desc_file is not None:
131 raise PackageError("Package contains more than one descriptor")
132 desc_file = filename
133
134 if desc_file is None:
135 raise PackageError("Could not find descriptor file in package")
136
137 return desc_file
138
139 @property
140 def descriptor_msg(self):
141 """ The proto-GI descriptor message """
142 filename = self.descriptor_file
143 with self.open(filename) as hdl:
144 _, ext = os.path.splitext(filename)
145 nsd = self.serializer.from_file_hdl(hdl, ext)
146 return nsd
147
148 @property
149 def json_descriptor(self):
150 """ The JSON serialized descriptor message"""
151 nsd = self.descriptor_msg
152 return self.serializer.to_json_string(nsd)
153
154 @property
155 def descriptor_id(self):
156 """ The descriptor id which uniquely identifies this descriptor in the system """
157 if not self.descriptor_msg.has_field("id"):
158 msg = "Descriptor must have an id field"
159 self._log.error(msg)
160 raise PackageError(msg)
161
162 return self.descriptor_msg.id
163
164 @property
165 def descriptor_name(self):
166 """ The descriptor name of this descriptor in the system """
167 if not self.descriptor_msg.has_field("name"):
168 msg = "Descriptor name not present"
169 self._log.error(msg)
170 raise PackageError(msg)
171
172 return self.descriptor_msg.name
173
174 @property
175 def descriptor_version(self):
176 desc_msg = self.descriptor_msg
177 return desc_msg.version if desc_msg.has_field("version") else ''
178
179 @property
180 def descriptor_vendor(self):
181 desc_msg = self.descriptor_msg
182 return desc_msg.vendor if desc_msg.has_field("vendor") else ''
183
184 @classmethod
185 def get_descriptor_patterns(cls):
186 """ Returns a tuple of descriptor regex and Package Types """
187 package_types = (VnfdPackage, NsdPackage)
188 patterns = []
189
190 for pkg_cls in package_types:
191 regex = cls.DESCRIPTOR_REGEX.format(
192 descriptor_type=pkg_cls.DESCRIPTOR_TYPE,
193 prefix=".*"
194 )
195
196 patterns.append((regex, pkg_cls))
197
198 return patterns
199
200 @classmethod
201 def from_package_files(cls, log, open_fn, files):
202 """ Creates a new DescriptorPackage subclass instance from a list of files
203
204 This classmethod detects the Package type from the package contents
205 and returns a new Package instance.
206
207 This will NOT subsequently add the files to the package so that must
208 be done by the client
209
210 Arguments:
211 log - A logger
212 open_fn - A function which can take a file name and mode and return
213 a file handle.
214 files - A list of files which would be added to the package after
215 intantiation
216
217 Returns:
218 A new DescriptorPackage subclass of the correct type for the descriptor
219
220 Raises:
221 PackageError - Package type could not be determined from the list of files.
222 """
223 patterns = cls.get_descriptor_patterns()
224 pkg_cls = None
225 regexes = set()
226 for name in files:
227 for regex, cls in patterns:
228 regexes.add(regex)
229 if re.match(regex, name) is not None:
230 pkg_cls = cls
231 break
232
233 if pkg_cls is None:
234 log.error("No file in archive matched known descriptor formats: %s", regexes)
235 raise PackageError("Could not determine package type from contents")
236
237 package = pkg_cls(log, open_fn)
238 return package
239
240 @classmethod
241 def from_descriptor_file_hdl(cls, log, file_hdl):
242 """ Creates a new DescriptorPackage from a descriptor file handle
243
244 The descriptor file is added to the package before returning.
245
246 Arguments:
247 log - A logger
248 file_hdl - A file handle whose name attribute can be recognized as
249 particular descriptor type.
250
251 Returns:
252 A new DescriptorPackage subclass of the correct type for the descriptor
253
254 Raises:
255 PackageError - Package type could not be determined from the list of files.
256 ValueError - file_hdl did not have a name attribute provided
257 """
258
259 package_types = (VnfdPackage, NsdPackage)
260 filename_patterns = []
261 for package_cls in package_types:
262 filename_patterns.append(
263 (r".*{}.*".format(package_cls.DESCRIPTOR_TYPE), package_cls)
264 )
265
266 if not hasattr(file_hdl, 'name'):
267 raise ValueError("File descriptor must have a name attribute to create a descriptor package")
268
269 # Iterate through the recognized patterns and assign files accordingly
270 package_cls = None
271 for pattern, cls in filename_patterns:
272 if re.match(pattern, file_hdl.name):
273 package_cls = cls
274 break
275
276 if not package_cls:
277 raise PackageError("Could not determine package type from file name: %s" % file_hdl.name)
278
279 _, ext = os.path.splitext(file_hdl.name)
280 try:
281 package_cls.SERIALIZER.from_file_hdl(file_hdl, ext)
282 except convert.SerializationError as e:
283 raise PackageError("Could not deserialize descriptor %s" % file_hdl.name) from e
284
285 # Create a new file handle for each open call to prevent independent clients
286 # from affecting each other
287 file_hdl.seek(0)
288 new_hdl = io.BytesIO(file_hdl.read())
289
290 def do_open(file_path):
291 assert file_path == file_hdl.name
292 hdl = io.BytesIO(new_hdl.getvalue())
293 return hdl
294
295 desc_pkg = package_cls(log, do_open)
296 desc_pkg.add_file(file_hdl.name)
297
298 return desc_pkg
299
300 def get_file_mode(self, pkg_file):
301 """ Returns the file mode for the package file
302
303 Arguments:
304 pkg_file - A file name in the package
305
306 Returns:
307 The permission mode
308
309 Raises:
310 PackageError - The file does not exist in the package
311 """
312 try:
313 return self._package_file_mode_map[pkg_file]
314 except KeyError as e:
315 msg = "Could not find package_file: %s" % pkg_file
316 self._log.error(msg)
317 raise PackageError(msg) from e
318
319 def extract_dir(self, src_dir, dest_root_dir, extract_images=False):
320 """ Extract a specific directory contents to dest_root_dir
321
322 Arguments:
323 src_dir - A directory within the package (None means all files/directories)
324 dest_root_dir - A directory to extract directory contents to
325 extract_images - A flag indicating whether we want to extract images
326
327 Raises:
328 ExtractError - Directory contents could not be extracted
329 """
330 if src_dir is not None and src_dir not in self._package_dirs:
331 raise ExtractError("Could not find source dir: %s" % src_dir)
332
333 for filename in self.files:
334 if not extract_images and image.is_image_file(filename):
335 continue
336
337 if src_dir is not None and not filename.startswith(src_dir):
338 continue
339
340 # Copy the contents of the file to the correct path
341 # Remove the common prefix and create the dest filename
342 if src_dir is not None:
343 fname = filename[len(src_dir):]
344 if fname[0] == '/':
345 fname = fname[1:]
346 else:
347 fname = filename
348 dest_file_path = os.path.join(dest_root_dir, fname)
349 dest_dir_path = os.path.dirname(dest_file_path)
350 if not os.path.exists(dest_dir_path):
351 os.makedirs(dest_dir_path)
352
353 with open(dest_file_path, 'wb') as dst_hdl:
354 with self.open(filename) as src_hdl:
355 shutil.copyfileobj(src_hdl, dst_hdl, 10 * 1024 * 1024)
356
357 # Set the file mode to original
358 os.chmod(dest_file_path, self._package_file_mode_map[filename])
359
360 def insert_file(self, new_file, dest_file, rel_path, mode=0o777):
361 self.add_file(rel_path, mode)
362
363 try:
364 # Copy the contents of the file to the correct path
365 # For folder creation (or nested folders), dest_file appears w/ trailing "/" like: dir1/ or dir1/dir2/
366 # For regular file upload, dest_file appears as dir1/abc.txt
367
368 dest_dir_path = os.path.dirname(dest_file)
369 if not os.path.isdir(dest_dir_path):
370 os.makedirs(dest_dir_path)
371 if not os.path.basename(dest_file):
372 self._log.debug("Created dir path, no filename to insert in {}, skipping..".format(dest_dir_path))
373 return
374
375 with open(dest_file, 'wb') as dst_hdl:
376 with open(new_file, 'rb') as src_hdl:
377 shutil.copyfileobj(src_hdl, dst_hdl, 10 * 1024 * 1024)
378
379 # Set the file mode to original
380 os.chmod(dest_file, self._package_file_mode_map[rel_path])
381 except Exception as e:
382 # Clear the file when an exception happens
383 if os.path.isfile(dest_file):
384 os.remove(dest_file)
385
386 raise PackageAppendError(str(e))
387
388 def delete_file(self, dest_file, rel_path):
389 self.remove_file(rel_path)
390
391 try:
392 os.remove(dest_file)
393 except Exception as e:
394 raise PackageAppendError(str(e))
395
396 def extract_file(self, src_file, dest_file):
397 """ Extract a specific package file to dest_file
398
399 The destination directory will be created if it does not exist.
400
401 Arguments:
402 src_file - A file within the package
403 dest_file - A file path to extract file contents to
404
405 Raises:
406 ExtractError - Directory contents could not be extracted
407 """
408 if src_file not in self._package_file_mode_map:
409 msg = "Could not find source file %s" % src_file
410 self._log.error(msg)
411 raise ExtractError(msg)
412
413 # Copy the contents of the file to the correct path
414 dest_dir_path = os.path.dirname(dest_file)
415 if not os.path.isdir(dest_dir_path):
416 os.makedirs(dest_dir_path)
417
418 with open(dest_file, 'wb') as dst_hdl:
419 with self.open(src_file) as src_hdl:
420 shutil.copyfileobj(src_hdl, dst_hdl, 10 * 1024 * 1024)
421
422 # Set the file mode to original
423 os.chmod(dest_file, self._package_file_mode_map[src_file])
424
425 def extract(self, dest_root_dir, extract_images=False):
426 """ Extract all package contents to a destination directory
427
428 Arguments:
429 dest_root_dir - The directory to extract package contents to
430
431 Raises:
432 NotADirectoryError - dest_root_dir is not a directory
433 """
434 if not os.path.isdir(dest_root_dir):
435 raise NotADirectoryError(dest_root_dir)
436
437 self.extract_dir(None, dest_root_dir, extract_images)
438
439 def open(self, rel_path):
440 """ Open a file contained in the package in read-only, binary mode.
441
442 Arguments:
443 rel_path - The file path within the package
444
445 Returns:
446 A file-like object opened in read-only mode.
447
448 Raises:
449 PackageError - The file could not be opened
450 """
451 try:
452 return self._open_fn(rel_path)
453 except Exception as e:
454 msg = "Could not open file from package: %s" % rel_path
455 self._log.warning(msg)
456 raise PackageError(msg) from e
457
458 def add_file(self, rel_path, mode=0o777):
459 """ Add a file to the package.
460
461 The file should be specified as a relative path to the package
462 root. The open_fn provided in the constructor must be able to
463 take the relative path and open the actual source file from
464 wherever the file actually is stored.
465
466 If the file's parent directories do not yet exist, add them to
467 the package.
468
469 Arguments:
470 rel_path - The file path relative to the top of the package.
471 mode - The permission mode the file should be stored with so
472 it can be extracted with the correct permissions.
473
474 Raises:
475 PackageError - The file could not be added to the package
476 """
477 if not rel_path:
478 raise PackageError("Empty file name added")
479
480 if rel_path in self._package_file_mode_map:
481 raise PackageError("File %s already exists in package" % rel_path)
482
483 # If the file's directory is not in the package add it.
484 rel_dir = os.path.dirname(rel_path)
485 while rel_dir:
486 self._package_dirs.add(rel_dir)
487 rel_dir = os.path.dirname(rel_dir)
488
489 self._package_file_mode_map[rel_path] = mode
490
491 def remove_file(self, rel_path):
492 if not rel_path:
493 raise PackageError("Empty file name added")
494
495 if rel_path not in self._package_file_mode_map:
496 raise PackageError("File %s does not exist in package" % rel_path)
497
498 del self._package_file_mode_map[rel_path]
499
500 def add_dir(self, rel_path):
501 """ Add a directory to the package
502
503 Arguments:
504 rel_path - The directories relative path.
505
506 Raises:
507 PackageError - A file already exists in the package with the same name.
508 """
509 if rel_path in self._package_file_mode_map:
510 raise PackageError("File already exists with the same name: %s", rel_path)
511
512 if rel_path in self._package_dirs:
513 self._log.warning("%s directory already exists", rel_path)
514 return
515
516 self._package_dirs.add(rel_path)
517
518
519 class NsdPackage(DescriptorPackage):
520 DESCRIPTOR_TYPE = "nsd"
521 SERIALIZER = convert.RwNsdSerializer()
522
523 @property
524 def descriptor_type(self):
525 return "nsd"
526
527 @property
528 def serializer(self):
529 return NsdPackage.SERIALIZER
530
531
532 class VnfdPackage(DescriptorPackage):
533 DESCRIPTOR_TYPE = "vnfd"
534 SERIALIZER = convert.RwVnfdSerializer()
535
536 @property
537 def descriptor_type(self):
538 return "vnfd"
539
540 @property
541 def serializer(self):
542 return VnfdPackage.SERIALIZER
543
544 class PackageConstructValidator(object):
545
546 def __init__(self, log):
547 self._log = log
548
549 def validate(self, package):
550 """ Validate presence of descriptor file (.yaml) at the top level in the
551 package folder structure.
552
553 Arguments:
554 package - The Descriptor Package being validated.
555 Returns:
556 None
557 Raises:
558 PackageValidationError - The package validation failed for some
559 generic reason.
560 """
561 pass
562 desc_file = package.descriptor_file
563 prefix, desc_file = package.prefix.rstrip('/'), desc_file.rstrip('/')
564
565 if os.path.dirname(desc_file) != prefix:
566 msg = "Descriptor file {} not found in expcted location {}".format(desc_file, prefix)
567 self._log.error(msg)
568 raise PackageValidationError(msg)
569
570
571 class PackageChecksumValidator(object):
572 """ This class uses the checksums.txt file in the package
573 and validates that all files in the package match the checksum that exists within
574 the file.
575 """
576 CHECKSUM_FILE = "{prefix}checksums.txt"
577
578 def __init__(self, log):
579 self._log = log
580 self.validated_file_checksums = {}
581
582 @classmethod
583 def get_package_checksum_file(cls, package):
584 checksum_file = cls.CHECKSUM_FILE.format(prefix=package.prefix)
585 if checksum_file not in package.files:
586 raise FileNotFoundError("%s does not exist in archive" % checksum_file)
587
588 return checksum_file
589
590 @property
591 def checksums(self):
592 return self.validated_file_checksums
593
594 def validate(self, package):
595 """ Validate file checksums match that in the checksums.txt
596
597 Arguments:
598 package - The Descriptor Package which possiblity contains checksums.txt
599
600 Returns: A dictionary of files that were validated by the checksums.txt
601 along with their checksums
602
603 Raises:
604 PackageValidationError - The package validation failed for some
605 generic reason.
606 PackageFileChecksumError - A file within the package did not match the
607 checksum within checksums.txt
608 """
609
610 try:
611 checksum_file = PackageChecksumValidator.get_package_checksum_file(package)
612 with package.open(checksum_file) as checksum_hdl:
613 archive_checksums = checksums.ArchiveChecksums.from_file_desc(checksum_hdl)
614 except (FileNotFoundError, PackageError) as e:
615 self._log.warning("Could not open package checksum file. Not validating checksums.")
616 return self.validated_file_checksums
617
618 for pkg_file in package.files:
619 if pkg_file == checksum_file:
620 continue
621
622 pkg_file_no_prefix = pkg_file.replace(package.prefix, "", 1)
623 if pkg_file_no_prefix not in archive_checksums:
624 self._log.warning("File %s not found in checksum file %s",
625 pkg_file, checksum_file)
626 continue
627
628 try:
629 with package.open(pkg_file) as pkg_file_hdl:
630 file_checksum = checksums.checksum(pkg_file_hdl)
631 except PackageError as e:
632 msg = "Could not read package file {} for checksum validation: {}".format(
633 pkg_file, str(e))
634 self._log.error(msg)
635 raise PackageValidationError(msg) from e
636
637 if archive_checksums[pkg_file_no_prefix] != file_checksum:
638 msg = "{} checksum ({}) did match expected checksum ({})".format(
639 pkg_file, file_checksum, archive_checksums[pkg_file_no_prefix]
640 )
641 self._log.error(msg)
642 raise PackageFileChecksumError(pkg_file)
643
644 self.validated_file_checksums[pkg_file] = file_checksum
645
646
647 class TarPackageArchive(object):
648 """ This class represents a package stored within a tar.gz archive file """
649 def __init__(self, log, tar_file_hdl, mode="r"):
650 self._log = log
651 self._tar_filepath = tar_file_hdl
652 self._tar_infos = {}
653
654 self._tarfile = tarfile.open(fileobj=tar_file_hdl, mode=mode)
655
656 self._load_archive()
657
658 def __repr__(self):
659 return "TarPackageArchive(%s)" % self._tar_filepath
660
661 def _get_members(self):
662 return [info for info in self._tarfile.getmembers()]
663
664 def _load_archive(self):
665 self._tar_infos = {info.name: info for info in self._get_members() if info.name}
666
667 def __del__(self):
668 self.close()
669
670 def close(self):
671 """ Close the opened tarfile"""
672 if self._tarfile is not None:
673 self._tarfile.close()
674 self._tarfile = None
675
676 @property
677 def filenames(self):
678 """ The list of file members within the tar file """
679 return [name for name in self._tar_infos if tarfile.TarInfo.isfile(self._tar_infos[name])]
680
681 def open_file(self, rel_file_path):
682 """ Opens a file within the archive as read-only, byte mode.
683
684 Arguments:
685 rel_file_path - The file path within the archive to open
686
687 Returns:
688 A file like object (see tarfile.extractfile())
689
690 Raises:
691 ArchiveError - The file could not be opened for some generic reason.
692 """
693 if rel_file_path not in self._tar_infos:
694 raise ArchiveError("Could not find %s in tar file", rel_file_path)
695
696 try:
697 return self._tarfile.extractfile(rel_file_path)
698 except tarfile.TarError as e:
699 msg = "Failed to read file {} from tarfile {}: {}".format(
700 rel_file_path, self._tar_filepath, str(e)
701 )
702 self._log.error(msg)
703 raise ArchiveError(msg) from e
704
705 def create_package(self):
706 """ Creates a Descriptor package from the archive contents
707
708 Returns:
709 A DescriptorPackage of the correct descriptor type
710 """
711 package = DescriptorPackage.from_package_files(self._log, self.open_file, self.filenames)
712 for pkg_file in self.filenames:
713 package.add_file(pkg_file, self._tar_infos[pkg_file].mode)
714
715 return package
716
717
718 class TemporaryPackage(object):
719 """ This class is a container for a temporary file-backed package
720
721 This class contains a DescriptorPackage and can be used in place of one.
722 Provides a useful context manager which will close and destroy the file
723 that is backing the DescriptorPackage on exit.
724 """
725 def __init__(self, log, package, file_hdl):
726 self._log = log
727 self._package = package
728 self._file_hdl = file_hdl
729
730 if not hasattr(self._file_hdl, "name"):
731 raise ValueError("File handle must have a name attribute")
732
733 def __getattr__(self, attr):
734 return getattr(self._package, attr)
735
736 def __enter__(self):
737 return self._package
738
739 def __exit__(self, type, value, tb):
740 self.close()
741
742 def filename(self):
743 """ Returns the filepath with is backing the Package """
744 return self._file_hdl.name
745
746 def package(self):
747 """ The contained DescriptorPackage instance """
748 return self._package
749
750 def close(self):
751 """ Close and remove the backed file """
752 filename = self._file_hdl.name
753
754 try:
755 self._file_hdl.close()
756 except OSError as e:
757 self._log.warning("Failed to close package file: %s", str(e))
758
759 try:
760 os.remove(filename)
761 except OSError as e:
762 self._log.warning("Failed to remove package file: %s", str(e))