osm_common/sol004_package.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Copyright 2020 Whitestack, LLC
   4 # *************************************************************
   5 #
   6 # This file is part of OSM common repository.
   7 # All Rights Reserved to Whitestack, LLC
   8 #
   9 # Licensed under the Apache License, Version 2.0 (the "License"); you may
  10 # not use this file except in compliance with the License. You may obtain
  11 # a copy of the License at
  12 #
  13 #         http://www.apache.org/licenses/LICENSE-2.0
  14 #
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  17 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  18 # License for the specific language governing permissions and limitations
  19 # under the License.
  20 #
  21 # For those usages not covered by the Apache License, Version 2.0 please
  22 # contact: agarcia@whitestack.com
  23 ##
  24
  25 """Python module for interacting with ETSI GS NFV-SOL004 compliant packages
  26
  27 This module provides a SOL004Package class for validating and interacting with
  28 ETSI SOL004 packages. A valid SOL004 package may have its files arranged according
  29 to one of the following two structures:
  30
  31 SOL004 with metadata directory    SOL004 without metadata directory
  32
  33 native_charm_vnf/                 native_charm_vnf/
  34 ├── TOSCA-Metadata                ├── native_charm_vnfd.mf
  35 │   └── TOSCA.meta                ├── native_charm_vnfd.yaml
  36 ├── manifest.mf                   ├── ChangeLog.txt
  37 ├── Definitions                   ├── Licenses
  38 │   └── native_charm_vnfd.yaml    │   └── license.lic
  39 ├── Files                         ├── Files
  40 │   ├── icons                     │   └── icons
  41 │   │   └── osm.png               │       └── osm.png
  42 │   ├── Licenses                  └── Scripts
  43 │   │   └── license.lic               ├── cloud_init
  44 │   └── changelog.txt                 │   └── cloud-config.txt
  45 └── Scripts                           └── charms
  46     ├── cloud_init                        └── simple
  47     │   └── cloud-config.txt                  ├── config.yaml
  48     └── charms                                ├── hooks
  49         └── simple                            │   ├── install
  50             ├── config.yaml                  ...
  51             ├── hooks                         │
  52             │   ├── install                   └── src
  53            ...                                    └── charm.py
  54             └── src
  55                 └── charm.py
  56 """
  57
  58 import yaml
  59 import os
  60 import hashlib
  61
  62
  63 _METADATA_FILE_PATH = 'TOSCA-Metadata/TOSCA.meta'
  64 _METADATA_DESCRIPTOR_FIELD = 'Entry-Definitions'
  65 _METADATA_MANIFEST_FIELD = 'ETSI-Entry-Manifest'
  66 _METADATA_CHANGELOG_FIELD = 'ETSI-Entry-Change-Log'
  67 _METADATA_LICENSES_FIELD = 'ETSI-Entry-Licenses'
  68 _METADATA_DEFAULT_CHANGELOG_PATH = 'ChangeLog.txt'
  69 _METADATA_DEFAULT_LICENSES_PATH = 'Licenses'
  70 _MANIFEST_FILE_PATH_FIELD = 'Source'
  71 _MANIFEST_FILE_HASH_ALGORITHM_FIELD = 'Algorithm'
  72 _MANIFEST_FILE_HASH_DIGEST_FIELD = 'Hash'
  73
  74
  75 class SOL004PackageException(Exception):
  76     pass
  77
  78
  79 class SOL004Package:
  80     def __init__(self, package_path=''):
  81         self._package_path = package_path
  82         self._package_metadata = self._parse_package_metadata()
  83         self._manifest_data = self._parse_manifest_data()
  84
  85     def _parse_package_metadata(self):
  86         try:
  87             return self._parse_package_metadata_with_metadata_dir()
  88         except FileNotFoundError:
  89             return self._parse_package_metadata_without_metadata_dir()
  90
  91     def _parse_package_metadata_with_metadata_dir(self):
  92         try:
  93             return self._parse_file_in_blocks(_METADATA_FILE_PATH)
  94         except FileNotFoundError as e:
  95             raise e
  96         except (Exception, OSError) as e:
  97             raise SOL004PackageException('Error parsing {}: {}'.format(_METADATA_FILE_PATH, e))
  98
  99     def _parse_package_metadata_without_metadata_dir(self):
 100         package_root_files = {f for f in os.listdir(self._package_path)}
 101         package_root_yamls = [f for f in package_root_files if f.endswith('.yml') or f.endswith('.yaml')]
 102         if len(package_root_yamls) != 1:
 103             error_msg = 'Error parsing package metadata: there should be exactly 1 descriptor YAML, found {}'
 104             raise SOL004PackageException(error_msg.format(len(package_root_yamls)))
 105         # TODO: Parse extra metadata from descriptor YAML?
 106         return [{
 107             _METADATA_DESCRIPTOR_FIELD: package_root_yamls[0],
 108             _METADATA_MANIFEST_FIELD: '{}.mf'.format(os.path.splitext(package_root_yamls[0])[0]),
 109             _METADATA_CHANGELOG_FIELD: _METADATA_DEFAULT_CHANGELOG_PATH,
 110             _METADATA_LICENSES_FIELD: _METADATA_DEFAULT_LICENSES_PATH
 111         }]
 112
 113     def _parse_manifest_data(self):
 114         manifest_path = None
 115         for tosca_meta in self._package_metadata:
 116             if _METADATA_MANIFEST_FIELD in tosca_meta:
 117                 manifest_path = tosca_meta[_METADATA_MANIFEST_FIELD]
 118                 break
 119         else:
 120             error_msg = 'Error parsing {}: no {} field on path'.format(_METADATA_FILE_PATH, _METADATA_MANIFEST_FIELD)
 121             raise SOL004PackageException(error_msg)
 122
 123         try:
 124             return self._parse_file_in_blocks(manifest_path)
 125         except (Exception, OSError) as e:
 126             raise SOL004PackageException('Error parsing {}: {}'.format(manifest_path, e))
 127
 128     def _get_package_file_full_path(self, file_relative_path):
 129         return os.path.join(self._package_path, file_relative_path)
 130
 131     def _parse_file_in_blocks(self, file_relative_path):
 132         file_path = self._get_package_file_full_path(file_relative_path)
 133         with open(file_path) as f:
 134             blocks = f.read().split('\n\n')
 135         parsed_blocks = map(yaml.safe_load, blocks)
 136         return [block for block in parsed_blocks if block is not None]
 137
 138     def _get_package_file_manifest_data(self, file_relative_path):
 139         for file_data in self._manifest_data:
 140             if file_data.get(_MANIFEST_FILE_PATH_FIELD, '') == file_relative_path:
 141                 return file_data
 142
 143         error_msg = 'Error parsing {} manifest data: file not found on manifest file'.format(file_relative_path)
 144         raise SOL004PackageException(error_msg)
 145
 146     def get_package_file_hash_digest_from_manifest(self, file_relative_path):
 147         """Returns the hash digest of a file inside this package as specified on the manifest file."""
 148         file_manifest_data = self._get_package_file_manifest_data(file_relative_path)
 149         try:
 150             return file_manifest_data[_MANIFEST_FILE_HASH_DIGEST_FIELD]
 151         except Exception as e:
 152             raise SOL004PackageException('Error parsing {} hash digest: {}'.format(file_relative_path, e))
 153
 154     def get_package_file_hash_algorithm_from_manifest(self, file_relative_path):
 155         """Returns the hash algorithm of a file inside this package as specified on the manifest file."""
 156         file_manifest_data = self._get_package_file_manifest_data(file_relative_path)
 157         try:
 158             return file_manifest_data[_MANIFEST_FILE_HASH_ALGORITHM_FIELD]
 159         except Exception as e:
 160             raise SOL004PackageException('Error parsing {} hash digest: {}'.format(file_relative_path, e))
 161
 162     @staticmethod
 163     def _get_hash_function_from_hash_algorithm(hash_algorithm):
 164         function_to_algorithm = {
 165             'SHA-256': hashlib.sha256,
 166             'SHA-512': hashlib.sha512
 167         }
 168         if hash_algorithm not in function_to_algorithm:
 169             error_msg = 'Error checking hash function: hash algorithm {} not supported'.format(hash_algorithm)
 170             raise SOL004PackageException(error_msg)
 171         return function_to_algorithm[hash_algorithm]
 172
 173     def _calculate_file_hash(self, file_relative_path, hash_algorithm):
 174         file_path = self._get_package_file_full_path(file_relative_path)
 175         hash_function = self._get_hash_function_from_hash_algorithm(hash_algorithm)
 176         try:
 177             with open(file_path, "rb") as f:
 178                 return hash_function(f.read()).hexdigest()
 179         except Exception as e:
 180             raise SOL004PackageException('Error hashing {}: {}'.format(file_relative_path, e))
 181
 182     def validate_package_file_hash(self, file_relative_path):
 183         """Validates the integrity of a file using the hash algorithm and digest on the package manifest."""
 184         hash_algorithm = self.get_package_file_hash_algorithm_from_manifest(file_relative_path)
 185         file_hash = self._calculate_file_hash(file_relative_path, hash_algorithm)
 186         expected_file_hash = self.get_package_file_hash_digest_from_manifest(file_relative_path)
 187         if file_hash != expected_file_hash:
 188             error_msg = 'Error validating {} hash: calculated hash {} is different than manifest hash {}'
 189             raise SOL004PackageException(error_msg.format(file_relative_path, file_hash, expected_file_hash))
 190
 191     def validate_package_hashes(self):
 192         """Validates the integrity of all files listed on the package manifest."""
 193         for file_data in self._manifest_data:
 194             if _MANIFEST_FILE_PATH_FIELD in file_data:
 195                 file_relative_path = file_data[_MANIFEST_FILE_PATH_FIELD]
 196                 self.validate_package_file_hash(file_relative_path)
 197
 198     def get_descriptor_location(self):
 199         """Returns this package descriptor location as a relative path from the package root."""
 200         for tosca_meta in self._package_metadata:
 201             if _METADATA_DESCRIPTOR_FIELD in tosca_meta:
 202                 return tosca_meta[_METADATA_DESCRIPTOR_FIELD]
 203
 204         error_msg = 'Error: no {} entry found on {}'.format(_METADATA_DESCRIPTOR_FIELD, _METADATA_FILE_PATH)
 205         raise SOL004PackageException(error_msg)