Revert "Full Juju Charm support"
[osm/SO.git] / rwlaunchpad / plugins / rwlaunchpadtasklet / rift / package / package.py
1
2 #
3 # Copyright 2016 RIFT.IO Inc
4 #
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
8 #
9 # http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16 #
17
18 import io
19 import os
20 import re
21 import shutil
22 import tarfile
23
24 from . import checksums
25 from . import convert
26 from . import image
27
28
29 class ArchiveError(Exception):
30 pass
31
32
33 class ExtractError(Exception):
34 pass
35
36
37 class PackageError(Exception):
38 pass
39
40
41 class PackageValidationError(Exception):
42 pass
43
44
45 class PackageAppendError(Exception):
46 pass
47
48
49 class PackageFileChecksumError(PackageValidationError):
50 def __init__(self, filename):
51 self.filename = filename
52 super().__init__("Checksum mismatch for {}".format(filename))
53
54
55 class DescriptorPackage(object):
56 """ This class provides an base class for a descriptor package representing
57
58 A descriptor package is a package which contains a single descriptor and any
59 associated files (logos, charms, scripts, etc). This package representation
60 attempts to be agnostic as to where the package files are being stored
61 (in memory, on disk, etc).
62
63 The package provides a simple interface to interact with the files within the
64 package and access the contained descriptor.
65 """
66 DESCRIPTOR_REGEX = r"{prefix}({descriptor_type}/[^/]*|[^/]*{descriptor_type})\.(xml|yml|yaml|json)$"
67
68 def __init__(self, log, open_fn):
69 self._log = log
70 self._open_fn = open_fn
71
72 self._package_file_mode_map = {}
73 self._package_dirs = set()
74
75 @property
76 def prefix(self):
77 """ Return the leading parent directories shared by all files in the package
78
79 In order to remain flexible as to where tar was invoked to create the package,
80 the prefix represents the common parent directory path which all files in the
81 package have in common.
82 """
83 entries = list(self._package_file_mode_map) + list(self._package_dirs)
84
85 if len(entries) > 1:
86 prefix = os.path.commonprefix(entries)
87 if prefix and not prefix.endswith("/"):
88 prefix += "/"
89 elif len(entries) == 1:
90 entry = entries[0]
91 if "/" in entry:
92 prefix = os.path.dirname(entry) + "/"
93 else:
94 prefix = ""
95 else:
96 prefix = ""
97
98 return prefix
99
100 @property
101 def files(self):
102 """ Return all files (with the prefix) in the package """
103 return list(self._package_file_mode_map)
104
105 @property
106 def dirs(self):
107 """ Return all directories in the package """
108 return list(self._package_dirs)
109
110 @property
111 def descriptor_type(self):
112 """ A shorthand name for the type of descriptor (e.g. nsd)"""
113 raise NotImplementedError("Subclass must implement this property")
114
115 @property
116 def serializer(self):
117 """ An instance of convert.ProtoMessageSerializer """
118 raise NotImplementedError("Subclass must implement this property")
119
120 @property
121 def descriptor_file(self):
122 """ The descriptor file name (with prefix) """
123 regex = self.__class__.DESCRIPTOR_REGEX.format(
124 descriptor_type=self.descriptor_type,
125 prefix=self.prefix,
126 )
127 desc_file = None
128 for filename in self.files:
129 if re.match(regex, filename):
130 if desc_file is not None:
131 raise PackageError("Package contains more than one descriptor")
132 desc_file = filename
133
134 if desc_file is None:
135 raise PackageError("Could not find descriptor file in package")
136
137 return desc_file
138
139 @property
140 def descriptor_msg(self):
141 """ The proto-GI descriptor message """
142 filename = self.descriptor_file
143 with self.open(filename) as hdl:
144 _, ext = os.path.splitext(filename)
145 nsd = self.serializer.from_file_hdl(hdl, ext)
146 return nsd
147
148 @property
149 def json_descriptor(self):
150 """ The JSON serialized descriptor message"""
151 nsd = self.descriptor_msg
152 return self.serializer.to_json_string(nsd)
153
154 @property
155 def descriptor_id(self):
156 """ The descriptor id which uniquely identifies this descriptor in the system """
157 if not self.descriptor_msg.has_field("id"):
158 msg = "Descriptor must have an id field"
159 self._log.error(msg)
160 raise PackageError(msg)
161
162 return self.descriptor_msg.id
163
164 @property
165 def descriptor_name(self):
166 """ The descriptor name of this descriptor in the system """
167 if not self.descriptor_msg.has_field("name"):
168 msg = "Descriptor name not present"
169 self._log.error(msg)
170 raise PackageError(msg)
171
172 return self.descriptor_msg.name
173
174 @property
175 def descriptor_version(self):
176 desc_msg = self.descriptor_msg
177 return desc_msg.version if desc_msg.has_field("version") else ''
178
179 @property
180 def descriptor_vendor(self):
181 desc_msg = self.descriptor_msg
182 return desc_msg.vendor if desc_msg.has_field("vendor") else ''
183
184 @classmethod
185 def get_descriptor_patterns(cls):
186 """ Returns a tuple of descriptor regex and Package Types """
187 package_types = (VnfdPackage, NsdPackage)
188 patterns = []
189
190 for pkg_cls in package_types:
191 regex = cls.DESCRIPTOR_REGEX.format(
192 descriptor_type=pkg_cls.DESCRIPTOR_TYPE,
193 prefix=".*"
194 )
195
196 patterns.append((regex, pkg_cls))
197
198 return patterns
199
200 @classmethod
201 def from_package_files(cls, log, open_fn, files):
202 """ Creates a new DescriptorPackage subclass instance from a list of files
203
204 This classmethod detects the Package type from the package contents
205 and returns a new Package instance.
206
207 This will NOT subsequently add the files to the package so that must
208 be done by the client
209
210 Arguments:
211 log - A logger
212 open_fn - A function which can take a file name and mode and return
213 a file handle.
214 files - A list of files which would be added to the package after
215 intantiation
216
217 Returns:
218 A new DescriptorPackage subclass of the correct type for the descriptor
219
220 Raises:
221 PackageError - Package type could not be determined from the list of files.
222 """
223 patterns = cls.get_descriptor_patterns()
224 pkg_cls = None
225 regexes = set()
226 for name in files:
227 for regex, cls in patterns:
228 regexes.add(regex)
229 if re.match(regex, name) is not None:
230 pkg_cls = cls
231 break
232
233 if pkg_cls is None:
234 log.error("No file in archive matched known descriptor formats: %s", regexes)
235 raise PackageError("Could not determine package type from contents")
236
237 package = pkg_cls(log, open_fn)
238 return package
239
240 @classmethod
241 def from_descriptor_file_hdl(cls, log, file_hdl):
242 """ Creates a new DescriptorPackage from a descriptor file handle
243
244 The descriptor file is added to the package before returning.
245
246 Arguments:
247 log - A logger
248 file_hdl - A file handle whose name attribute can be recognized as
249 particular descriptor type.
250
251 Returns:
252 A new DescriptorPackage subclass of the correct type for the descriptor
253
254 Raises:
255 PackageError - Package type could not be determined from the list of files.
256 ValueError - file_hdl did not have a name attribute provided
257 """
258
259 package_types = (VnfdPackage, NsdPackage)
260 filename_patterns = []
261 for package_cls in package_types:
262 filename_patterns.append(
263 (r".*{}.*".format(package_cls.DESCRIPTOR_TYPE), package_cls)
264 )
265
266 if not hasattr(file_hdl, 'name'):
267 raise ValueError("File descriptor must have a name attribute to create a descriptor package")
268
269 # Iterate through the recognized patterns and assign files accordingly
270 package_cls = None
271 for pattern, cls in filename_patterns:
272 if re.match(pattern, file_hdl.name):
273 package_cls = cls
274 break
275
276 if not package_cls:
277 raise PackageError("Could not determine package type from file name: %s" % file_hdl.name)
278
279 _, ext = os.path.splitext(file_hdl.name)
280 try:
281 package_cls.SERIALIZER.from_file_hdl(file_hdl, ext)
282 except convert.SerializationError as e:
283 raise PackageError("Could not deserialize descriptor %s" % file_hdl.name) from e
284
285 # Create a new file handle for each open call to prevent independent clients
286 # from affecting each other
287 file_hdl.seek(0)
288 new_hdl = io.BytesIO(file_hdl.read())
289
290 def do_open(file_path):
291 assert file_path == file_hdl.name
292 hdl = io.BytesIO(new_hdl.getvalue())
293 return hdl
294
295 desc_pkg = package_cls(log, do_open)
296 desc_pkg.add_file(file_hdl.name)
297
298 return desc_pkg
299
300 def get_file_mode(self, pkg_file):
301 """ Returns the file mode for the package file
302
303 Arguments:
304 pkg_file - A file name in the package
305
306 Returns:
307 The permission mode
308
309 Raises:
310 PackageError - The file does not exist in the package
311 """
312 try:
313 return self._package_file_mode_map[pkg_file]
314 except KeyError as e:
315 msg = "Could not find package_file: %s" % pkg_file
316 self._log.error(msg)
317 raise PackageError(msg) from e
318
319 def extract_dir(self, src_dir, dest_root_dir, extract_images=False):
320 """ Extract a specific directory contents to dest_root_dir
321
322 Arguments:
323 src_dir - A directory within the package (None means all files/directories)
324 dest_root_dir - A directory to extract directory contents to
325 extract_images - A flag indicating whether we want to extract images
326
327 Raises:
328 ExtractError - Directory contents could not be extracted
329 """
330 if src_dir is not None and src_dir not in self._package_dirs:
331 raise ExtractError("Could not find source dir: %s" % src_dir)
332
333 for filename in self.files:
334 if not extract_images and image.is_image_file(filename):
335 continue
336
337 if src_dir is not None and not filename.startswith(src_dir):
338 continue
339
340 # Copy the contents of the file to the correct path
341 # Remove the common prefix and create the dest filename
342 if src_dir is not None:
343 fname = filename[len(src_dir):]
344 if fname[0] == '/':
345 fname = fname[1:]
346 else:
347 fname = filename
348 dest_file_path = os.path.join(dest_root_dir, fname)
349 dest_dir_path = os.path.dirname(dest_file_path)
350 if not os.path.exists(dest_dir_path):
351 os.makedirs(dest_dir_path)
352
353 with open(dest_file_path, 'wb') as dst_hdl:
354 with self.open(filename) as src_hdl:
355 shutil.copyfileobj(src_hdl, dst_hdl, 10 * 1024 * 1024)
356
357 # Set the file mode to original
358 os.chmod(dest_file_path, self._package_file_mode_map[filename])
359
360 def insert_file(self, new_file, dest_file, rel_path, mode=0o777):
361 self.add_file(rel_path, mode)
362
363 try:
364 # Copy the contents of the file to the correct path
365 # For folder creation (or nested folders), dest_file appears w/ trailing "/" like: dir1/ or dir1/dir2/
366 # For regular file upload, dest_file appears as dir1/abc.txt
367
368 dest_dir_path = os.path.dirname(dest_file)
369 if not os.path.isdir(dest_dir_path):
370 os.makedirs(dest_dir_path)
371 if not os.path.basename(dest_file):
372 self._log.debug("Created dir path, no filename to insert in {}, skipping..".format(dest_dir_path))
373 return
374
375 with open(dest_file, 'wb') as dst_hdl:
376 with open(new_file, 'rb') as src_hdl:
377 shutil.copyfileobj(src_hdl, dst_hdl, 10 * 1024 * 1024)
378
379 # Set the file mode to original
380 os.chmod(dest_file, self._package_file_mode_map[rel_path])
381 except Exception as e:
382 # Clear the file when an exception happens
383 if os.path.isfile(dest_file):
384 os.remove(dest_file)
385
386 raise PackageAppendError(str(e))
387
388 def delete_file(self, dest_file, rel_path):
389 self.remove_file(rel_path)
390
391 try:
392 os.remove(dest_file)
393 except Exception as e:
394 raise PackageAppendError(str(e))
395
396 def extract_file(self, src_file, dest_file):
397 """ Extract a specific package file to dest_file
398
399 The destination directory will be created if it does not exist.
400
401 Arguments:
402 src_file - A file within the package
403 dest_file - A file path to extract file contents to
404
405 Raises:
406 ExtractError - Directory contents could not be extracted
407 """
408 if src_file not in self._package_file_mode_map:
409 msg = "Could not find source file %s" % src_file
410 self._log.error(msg)
411 raise ExtractError(msg)
412
413 # Copy the contents of the file to the correct path
414 dest_dir_path = os.path.dirname(dest_file)
415 if not os.path.isdir(dest_dir_path):
416 os.makedirs(dest_dir_path)
417
418 with open(dest_file, 'wb') as dst_hdl:
419 with self.open(src_file) as src_hdl:
420 shutil.copyfileobj(src_hdl, dst_hdl, 10 * 1024 * 1024)
421
422 # Set the file mode to original
423 os.chmod(dest_file, self._package_file_mode_map[src_file])
424
425 def extract(self, dest_root_dir, extract_images=False):
426 """ Extract all package contents to a destination directory
427
428 Arguments:
429 dest_root_dir - The directory to extract package contents to
430
431 Raises:
432 NotADirectoryError - dest_root_dir is not a directory
433 """
434 def find_prefix():
435 """ Find comon prefix of all files in package. This prefix will be
436 used to collapse directory structure during extraction to eliminate
437 empty nested folders.
438 """
439 common_dir = set()
440 for f in self.files:
441 common_dir.add(os.path.dirname(f))
442 prefix = os.path.commonprefix(list(common_dir))
443 return prefix if prefix else None
444
445 if not os.path.isdir(dest_root_dir):
446 raise NotADirectoryError(dest_root_dir)
447
448 self.extract_dir(find_prefix(), dest_root_dir, extract_images)
449
450 def open(self, rel_path):
451 """ Open a file contained in the package in read-only, binary mode.
452
453 Arguments:
454 rel_path - The file path within the package
455
456 Returns:
457 A file-like object opened in read-only mode.
458
459 Raises:
460 PackageError - The file could not be opened
461 """
462 try:
463 return self._open_fn(rel_path)
464 except Exception as e:
465 msg = "Could not open file from package: %s" % rel_path
466 self._log.warning(msg)
467 raise PackageError(msg) from e
468
469 def add_file(self, rel_path, mode=0o777):
470 """ Add a file to the package.
471
472 The file should be specified as a relative path to the package
473 root. The open_fn provided in the constructor must be able to
474 take the relative path and open the actual source file from
475 wherever the file actually is stored.
476
477 If the file's parent directories do not yet exist, add them to
478 the package.
479
480 Arguments:
481 rel_path - The file path relative to the top of the package.
482 mode - The permission mode the file should be stored with so
483 it can be extracted with the correct permissions.
484
485 Raises:
486 PackageError - The file could not be added to the package
487 """
488 if not rel_path:
489 raise PackageError("Empty file name added")
490
491 if rel_path in self._package_file_mode_map:
492 raise PackageError("File %s already exists in package" % rel_path)
493
494 # If the file's directory is not in the package add it.
495 rel_dir = os.path.dirname(rel_path)
496 while rel_dir:
497 self._package_dirs.add(rel_dir)
498 rel_dir = os.path.dirname(rel_dir)
499
500 self._package_file_mode_map[rel_path] = mode
501
502 def remove_file(self, rel_path):
503 if not rel_path:
504 raise PackageError("Empty file name added")
505
506 if rel_path not in self._package_file_mode_map:
507 raise PackageError("File %s does not exist in package" % rel_path)
508
509 del self._package_file_mode_map[rel_path]
510
511 def add_dir(self, rel_path):
512 """ Add a directory to the package
513
514 Arguments:
515 rel_path - The directories relative path.
516
517 Raises:
518 PackageError - A file already exists in the package with the same name.
519 """
520 if rel_path in self._package_file_mode_map:
521 raise PackageError("File already exists with the same name: %s", rel_path)
522
523 if rel_path in self._package_dirs:
524 self._log.warning("%s directory already exists", rel_path)
525 return
526
527 self._package_dirs.add(rel_path)
528
529
530 class NsdPackage(DescriptorPackage):
531 DESCRIPTOR_TYPE = "nsd"
532 SERIALIZER = convert.RwNsdSerializer()
533
534 @property
535 def descriptor_type(self):
536 return "nsd"
537
538 @property
539 def serializer(self):
540 return NsdPackage.SERIALIZER
541
542
543 class VnfdPackage(DescriptorPackage):
544 DESCRIPTOR_TYPE = "vnfd"
545 SERIALIZER = convert.RwVnfdSerializer()
546
547 @property
548 def descriptor_type(self):
549 return "vnfd"
550
551 @property
552 def serializer(self):
553 return VnfdPackage.SERIALIZER
554
555 class PackageConstructValidator(object):
556
557 def __init__(self, log):
558 self._log = log
559
560 def validate(self, package):
561 """ Validate presence of descriptor file (.yaml) at the top level in the
562 package folder structure.
563
564 Arguments:
565 package - The Descriptor Package being validated.
566 Returns:
567 None
568 Raises:
569 PackageValidationError - The package validation failed for some
570 generic reason.
571 """
572 pass
573 desc_file = package.descriptor_file
574 prefix, desc_file = package.prefix.rstrip('/'), desc_file.rstrip('/')
575
576 if os.path.dirname(desc_file) != prefix:
577 msg = "Descriptor file {} not found in expcted location {}".format(desc_file, prefix)
578 self._log.error(msg)
579 raise PackageValidationError(msg)
580
581
582 class PackageChecksumValidator(object):
583 """ This class uses the checksums.txt file in the package
584 and validates that all files in the package match the checksum that exists within
585 the file.
586 """
587 CHECKSUM_FILE = "{prefix}checksums.txt"
588
589 def __init__(self, log):
590 self._log = log
591 self.validated_file_checksums = {}
592
593 @classmethod
594 def get_package_checksum_file(cls, package):
595 checksum_file = cls.CHECKSUM_FILE.format(prefix=package.prefix)
596 if checksum_file not in package.files:
597 raise FileNotFoundError("%s does not exist in archive" % checksum_file)
598
599 return checksum_file
600
601 @property
602 def checksums(self):
603 return self.validated_file_checksums
604
605 def validate(self, package):
606 """ Validate file checksums match that in the checksums.txt
607
608 Arguments:
609 package - The Descriptor Package which possiblity contains checksums.txt
610
611 Returns: A dictionary of files that were validated by the checksums.txt
612 along with their checksums
613
614 Raises:
615 PackageValidationError - The package validation failed for some
616 generic reason.
617 PackageFileChecksumError - A file within the package did not match the
618 checksum within checksums.txt
619 """
620
621 try:
622 checksum_file = PackageChecksumValidator.get_package_checksum_file(package)
623 with package.open(checksum_file) as checksum_hdl:
624 archive_checksums = checksums.ArchiveChecksums.from_file_desc(checksum_hdl)
625 except (FileNotFoundError, PackageError) as e:
626 self._log.warning("Could not open package checksum file. Not validating checksums.")
627 return self.validated_file_checksums
628
629 for pkg_file in package.files:
630 if pkg_file == checksum_file:
631 continue
632
633 pkg_file_no_prefix = pkg_file.replace(package.prefix, "", 1)
634 if pkg_file_no_prefix not in archive_checksums:
635 self._log.warning("File %s not found in checksum file %s",
636 pkg_file, checksum_file)
637 continue
638
639 try:
640 with package.open(pkg_file) as pkg_file_hdl:
641 file_checksum = checksums.checksum(pkg_file_hdl)
642 except PackageError as e:
643 msg = "Could not read package file {} for checksum validation: {}".format(
644 pkg_file, str(e))
645 self._log.error(msg)
646 raise PackageValidationError(msg) from e
647
648 if archive_checksums[pkg_file_no_prefix] != file_checksum:
649 msg = "{} checksum ({}) did not match expected checksum ({})".format(
650 pkg_file, file_checksum, archive_checksums[pkg_file_no_prefix]
651 )
652 self._log.error(msg)
653 raise PackageFileChecksumError(pkg_file)
654
655 self.validated_file_checksums[pkg_file] = file_checksum
656
657
658 class TarPackageArchive(object):
659 """ This class represents a package stored within a tar.gz archive file """
660 def __init__(self, log, tar_file_hdl, mode="r"):
661 self._log = log
662 self._tar_filepath = tar_file_hdl
663 self._tar_infos = {}
664
665 self._tarfile = tarfile.open(fileobj=tar_file_hdl, mode=mode)
666
667 self._load_archive()
668
669 def __repr__(self):
670 return "TarPackageArchive(%s)" % self._tar_filepath
671
672 def _get_members(self):
673 return [info for info in self._tarfile.getmembers()]
674
675 def _load_archive(self):
676 self._tar_infos = {info.name: info for info in self._get_members() if info.name}
677
678 def __del__(self):
679 self.close()
680
681 def close(self):
682 """ Close the opened tarfile"""
683 if self._tarfile is not None:
684 self._tarfile.close()
685 self._tarfile = None
686
687 @property
688 def filenames(self):
689 """ The list of file members within the tar file """
690 return [name for name in self._tar_infos if tarfile.TarInfo.isfile(self._tar_infos[name])]
691
692 def open_file(self, rel_file_path):
693 """ Opens a file within the archive as read-only, byte mode.
694
695 Arguments:
696 rel_file_path - The file path within the archive to open
697
698 Returns:
699 A file like object (see tarfile.extractfile())
700
701 Raises:
702 ArchiveError - The file could not be opened for some generic reason.
703 """
704 if rel_file_path not in self._tar_infos:
705 raise ArchiveError("Could not find %s in tar file", rel_file_path)
706
707 try:
708 return self._tarfile.extractfile(rel_file_path)
709 except tarfile.TarError as e:
710 msg = "Failed to read file {} from tarfile {}: {}".format(
711 rel_file_path, self._tar_filepath, str(e)
712 )
713 self._log.error(msg)
714 raise ArchiveError(msg) from e
715
716 def create_package(self):
717 """ Creates a Descriptor package from the archive contents
718
719 Returns:
720 A DescriptorPackage of the correct descriptor type
721 """
722 package = DescriptorPackage.from_package_files(self._log, self.open_file, self.filenames)
723 for pkg_file in self.filenames:
724 package.add_file(pkg_file, self._tar_infos[pkg_file].mode)
725
726 return package
727
728
729 class TemporaryPackage(object):
730 """ This class is a container for a temporary file-backed package
731
732 This class contains a DescriptorPackage and can be used in place of one.
733 Provides a useful context manager which will close and destroy the file
734 that is backing the DescriptorPackage on exit.
735 """
736 def __init__(self, log, package, file_hdl):
737 self._log = log
738 self._package = package
739 self._file_hdl = file_hdl
740
741 if not hasattr(self._file_hdl, "name"):
742 raise ValueError("File handle must have a name attribute")
743
744 def __getattr__(self, attr):
745 return getattr(self._package, attr)
746
747 def __enter__(self):
748 return self._package
749
750 def __exit__(self, type, value, tb):
751 self.close()
752
753 def filename(self):
754 """ Returns the filepath with is backing the Package """
755 return self._file_hdl.name
756
757 def package(self):
758 """ The contained DescriptorPackage instance """
759 return self._package
760
761 def close(self):
762 """ Close and remove the backed file """
763 filename = self._file_hdl.name
764
765 try:
766 self._file_hdl.close()
767 except OSError as e:
768 self._log.warning("Failed to close package file: %s", str(e))
769
770 try:
771 os.remove(filename)
772 except OSError as e:
773 self._log.warning("Failed to remove package file: %s", str(e))