X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osmclient%2Fcommon%2Fpackage_tool.py;h=3da96d57e0013d6bf3f5207b049b724d25473c88;hb=5f8a12367e3fba2a6f2ee7025e801dbff6da37a7;hp=72d7d3dcfce5f942351bb0dc781fecfa54ffa8b8;hpb=566a8c34fbf1d676a0acd725f6f30fce8baf8c0a;p=osm%2Fosmclient.git diff --git a/osmclient/common/package_tool.py b/osmclient/common/package_tool.py index 72d7d3d..3da96d5 100644 --- a/osmclient/common/package_tool.py +++ b/osmclient/common/package_tool.py @@ -126,14 +126,16 @@ class PackageTool(object): self._logger.debug("") package_folder = package_folder.rstrip('/') if not os.path.exists("{}".format(package_folder)): - return "Fail, package is not in the specified route" + return "Fail, package is not in the specified path" if not skip_validation: + print('Validating package {}'.format(package_folder)) results = self.validate(package_folder, recursive=False) if results: for result in results: if result["valid"] != "OK": - raise ClientException("There was an error validating the file: {} with error: {}" + raise ClientException("There was an error validating the file {} with error: {}" .format(result["path"], result["error"])) + print('Validation OK') else: raise ClientException("No descriptor file found in: {}".format(package_folder)) charm_list = self.build_all_charms(package_folder, skip_charm_build) @@ -148,19 +150,18 @@ class PackageTool(object): :returns: None """ self._logger.debug("") - files = [f for f in glob.glob(package_folder + "/**/*.*", recursive=True)] - checksum = open("{}/checksums.txt".format(package_folder), "w+") - for file_item in files: - if "checksums.txt" in file_item: - continue - # from https://www.quickprogrammingtips.com/python/how-to-calculate-md5-hash-of-a-file-in-python.html - md5_hash = hashlib.md5() - with open(file_item, "rb") as f: - # Read and update hash in chunks of 4K - for byte_block in iter(lambda: f.read(4096), b""): - md5_hash.update(byte_block) - checksum.write("{}\t{}\n".format(md5_hash.hexdigest(), file_item)) - checksum.close() + files = [f for f in glob.glob(package_folder + "/**/*.*", recursive=True) if os.path.isfile(f)] + with open("{}/checksums.txt".format(package_folder), "w+") as checksum: + for file_item in files: + if "checksums.txt" in file_item: + continue + # from https://www.quickprogrammingtips.com/python/how-to-calculate-md5-hash-of-a-file-in-python.html + md5_hash = hashlib.md5() + with open(file_item, "rb") as f: + # Read and update hash in chunks of 4K + for byte_block in iter(lambda: f.read(4096), b""): + md5_hash.update(byte_block) + checksum.write("{}\t{}\n".format(md5_hash.hexdigest(), file_item)) def create_folders(self, folders, package_type): """ @@ -269,28 +270,32 @@ class PackageTool(object): - packet_folder: is the location of the package :return: Files and Folders not found. In case of override, it will return all file list """ - listCharms = [] self._logger.debug("") + listCharms = [] descriptor_file = False descriptors_paths = [f for f in glob.glob(package_folder + "/*.yaml")] for file in descriptors_paths: - if 'nfd.yaml' in file: + if file.endswith('nfd.yaml'): descriptor_file = True listCharms = self.charms_search(file, 'vnf') - if 'nsd.yaml' in file: + if file.endswith('nsd.yaml'): descriptor_file = True listCharms = self.charms_search(file, 'ns') + print("List of charms in the descriptor: {}".format(listCharms)) if not descriptor_file: raise ClientException ('descriptor name is not correct in: {}'.format(package_folder)) if listCharms and not skip_charm_build: for charmName in listCharms: if os.path.isdir('{}/charms/layers/{}'.format(package_folder,charmName)): + print('Building charm {}/charms/layers/{}'.format(package_folder, charmName)) self.charm_build(package_folder, charmName) + print('Charm built {}'.format(charmName)) else: if not os.path.isdir('{}/charms/{}'.format(package_folder,charmName)): raise ClientException ('The charm: {} referenced in the descriptor file ' 'is not present either in {}/charms or in {}/charms/layers'. format(charmName, package_folder,package_folder)) + self._logger.debug("Return list of charms: {}".format(listCharms)) return listCharms def discover_folder_structure(self, base_directory, name, override): @@ -341,11 +346,13 @@ class PackageTool(object): os.environ['CHARM_LAYERS_DIR'] = "{}/layers".format(os.environ['JUJU_REPOSITORY']) os.environ['CHARM_INTERFACES_DIR'] = "{}/interfaces".format(os.environ['JUJU_REPOSITORY']) os.environ['CHARM_BUILD_DIR'] = "{}/charms/builds".format(charms_folder) + if not os.path.exists(os.environ['CHARM_BUILD_DIR']): + os.makedirs(os.environ['CHARM_BUILD_DIR']) src_folder = '{}/{}'.format(os.environ['CHARM_LAYERS_DIR'], build_name) result = subprocess.run(["charm", "build", "{}".format(src_folder)]) if result.returncode == 1: raise ClientException("failed to build the charm: {}".format(src_folder)) - self._logger.verbose("charm: {} compiled".format(src_folder)) + self._logger.verbose("charm {} built".format(src_folder)) def build_tarfile(self, package_folder, charm_list=None): """ @@ -354,62 +361,78 @@ class PackageTool(object): returns: .tar.gz name """ self._logger.debug("") - ignore_patterns = "'*layers*', '*interfaces*'" + cwd = None try: - directory_name = self.create_temp_dir(package_folder, ignore_patterns, charm_list) + directory_name, package_name = self.create_temp_dir(package_folder, charm_list) cwd = os.getcwd() os.chdir(directory_name) - self.calculate_checksum(package_folder) - with tarfile.open("{}.tar.gz".format(package_folder), mode='w:gz') as archive: - print("Adding File: {}".format(package_folder)) - archive.add('{}'.format(package_folder), recursive=True) + self.calculate_checksum(package_name) + with tarfile.open("{}.tar.gz".format(package_name), mode='w:gz') as archive: + print("Adding File: {}".format(package_name)) + archive.add('{}'.format(package_name), recursive=True) #return "Created {}.tar.gz".format(package_folder) #self.build("{}".format(os.path.basename(package_folder))) os.chdir(cwd) - except: + except Exception as exc: + if cwd: + os.chdir(cwd) shutil.rmtree(os.path.join(package_folder, "tmp")) - raise ClientException('failure to manipulate the result of the compilation') - os.rename("{}/{}.tar.gz".format(directory_name, os.path.basename(package_folder)), - "{}.tar.gz".format(os.path.basename(package_folder))) - os.rename("{}/{}/checksums.txt".format(directory_name, os.path.basename(package_folder)), + raise ClientException('failure during build of targz file (create temp dir, calculate checksum, tar.gz file): {}'.format(exc)) + created_package = "{}/{}.tar.gz".format(package_folder, package_name) + os.rename("{}/{}.tar.gz".format(directory_name, package_name), + created_package) + os.rename("{}/{}/checksums.txt".format(directory_name, package_name), "{}/checksums.txt".format(package_folder)) shutil.rmtree(os.path.join(package_folder, "tmp")) - self._logger.verbose("package created: {}.tar.gz".format(os.path.basename(package_folder))) - return "{}.tar.gz".format(package_folder) + print("Package created: {}".format(created_package)) + return created_package - def create_temp_dir(self, package_folder, ignore_patterns=None, charm_list=None): + def create_temp_dir(self, package_folder, charm_list=None): """ - Method to create a temporary folder where we can move the files in package_folder, which do not - meet the pattern defined in ignore_patterns + Method to create a temporary folder where we can move the files in package_folder """ self._logger.debug("") + ignore_patterns = ('.gitignore') ignore = shutil.ignore_patterns(ignore_patterns) - os.makedirs("{}/tmp".format(package_folder), exist_ok=True) - directory_name = os.path.abspath("{}/tmp".format(package_folder)) - os.makedirs("{}/{}".format(directory_name, os.path.basename(package_folder),exist_ok=True)) + directory_name = os.path.abspath(package_folder) + package_name = os.path.basename(directory_name) + directory_name += "/tmp" + os.makedirs("{}/{}".format(directory_name, package_name), exist_ok=True) + self._logger.debug("Makedirs DONE: {}/{}".format(directory_name, package_name)) for item in os.listdir(package_folder): + self._logger.debug("Item: {}".format(item)) if item != "tmp": s = os.path.join(package_folder, item) - d = os.path.join(os.path.join(directory_name, os.path.basename(package_folder)), item) + d = os.path.join(os.path.join(directory_name, package_name), item) if os.path.isdir(s): if item == "charms": - s = os.path.join(s, "builds") - if not os.path.exists(s): - os.makedirs(s) - for i in os.listdir(s): - if i in charm_list: - s_charm = os.path.join(s, i) - # d_charm = os.path.join(package_folder, item, i) - d_temp = os.path.join(d, i) - # if os.path.exists(d_charm): - # shutil.rmtree(d_charm) - shutil.copytree(s_charm, d_temp, symlinks = True, ignore = ignore) - # shutil.copytree(s_charm, d_charm, symlinks = True, ignore = ignore) + os.makedirs(d, exist_ok=True) + s_builds = os.path.join(s, "builds") + for charm in charm_list: + self._logger.debug("Copying charm {}".format(charm)) + if charm in os.listdir(s): + s_charm = os.path.join(s, charm) + elif charm in os.listdir(s_builds): + s_charm = os.path.join(s_builds, charm) + else: + raise ClientException('The charm {} referenced in the descriptor file ' + 'could not be found in {}/charms or in {}/charms/builds'. + format(charm, package_folder, package_folder)) + d_temp = os.path.join(d, charm) + self._logger.debug("Copying tree: {} -> {}".format(s_charm, d_temp)) + shutil.copytree(s_charm, d_temp, symlinks=True, ignore=ignore) + self._logger.debug("DONE") else: - shutil.copytree(s, d, symlinks = True, ignore = ignore) + self._logger.debug("Copying tree: {} -> {}".format(s,d)) + shutil.copytree(s, d, symlinks=True, ignore=ignore) + self._logger.debug("DONE") else: + if item in ignore_patterns: + continue + self._logger.debug("Copying file: {} -> {}".format(s,d)) shutil.copy2(s, d) - return directory_name + self._logger.debug("DONE") + return directory_name, package_name def charms_search(self, descriptor_file, desc_type): self._logger.debug("")