X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;ds=sidebyside;f=common%2Fpython%2Frift%2Fmano%2Ftosca_translator%2Fshell.py;fp=common%2Fpython%2Frift%2Fmano%2Ftosca_translator%2Fshell.py;h=9221c79aad4f7ca24e1cb1bf3772bea481661948;hb=6f07e6f33f751ab4ffe624f6037f887b243bece2;hp=0000000000000000000000000000000000000000;hpb=72a563886272088feb7cb52e4aafbe6d2c580ff9;p=osm%2FSO.git diff --git a/common/python/rift/mano/tosca_translator/shell.py b/common/python/rift/mano/tosca_translator/shell.py new file mode 100644 index 00000000..9221c79a --- /dev/null +++ b/common/python/rift/mano/tosca_translator/shell.py @@ -0,0 +1,515 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Copyright 2016 RIFT.io Inc + + +import argparse +import logging +import logging.config +import os +import shutil +import stat +import subprocess +import tempfile +import zipfile + +import magic + +import yaml + +from rift.mano.tosca_translator.common.utils import _ +from rift.mano.tosca_translator.common.utils import ChecksumUtils +from rift.mano.tosca_translator.rwmano.syntax.mano_template import ManoTemplate +from rift.mano.tosca_translator.rwmano.tosca_translator import TOSCATranslator + +from toscaparser.tosca_template import ToscaTemplate + + +""" +Test the tosca translation from command line as: +#translator + --template-file= + --template-type= + --parameters="purpose=test" + --output_dir= + --archive + --validate_only +Takes following user arguments, +. Path to the file that needs to be translated (required) +. Input parameters (optional) +. Write to output files in a dir (optional), else print on screen +. Create archive or not + +In order to use translator to only validate template, +without actual translation, pass --validate-only along with +other required arguments. + +""" + + +class ToscaShellError(Exception): + pass + + +class ToscaEntryFileError(ToscaShellError): + pass + + +class ToscaNoEntryDefinitionError(ToscaShellError): + pass + + +class ToscaEntryFileNotFoundError(ToscaShellError): + pass + + +class ToscaCreateArchiveError(ToscaShellError): + pass + + +class TranslatorShell(object): + + SUPPORTED_TYPES = ['tosca'] + COPY_DIRS = ['images'] + SUPPORTED_INPUTS = (YAML, ZIP) = ('yaml', 'zip') + + def __init__(self, log=None): + self.log = log + + def main(self, raw_args=None): + args = self._parse_args(raw_args) + + if self.log is None: + if args.debug: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.ERROR) + self.log = logging.getLogger("tosca-translator") + + self.template_file = args.template_file + + parsed_params = {} + if args.parameters: + parsed_params = self._parse_parameters(args.parameters) + + self.archive = False + if args.archive: + self.archive = True + + self.tmpdir = None + + if args.validate_only: + a_file = os.path.isfile(args.template_file) + tpl = ToscaTemplate(self.template_file, parsed_params, a_file) + self.log.debug(_('Template = {}').format(tpl.__dict__)) + msg = (_('The input {} successfully passed ' \ + 'validation.').format(self.template_file)) + print(msg) + else: + self.use_gi = not args.no_gi + tpl = self._translate("tosca", parsed_params) + if tpl: + return self._write_output(tpl, args.output_dir) + + def translate(self, + template_file, + output_dir=None, + use_gi=True, + archive=False,): + self.template_file = template_file + + # Check the input file + path = os.path.abspath(template_file) + self.in_file = path + a_file = os.path.isfile(path) + if not a_file: + msg = _("The path {0} is not a valid file.").format(template_file) + self.log.error(msg) + raise ValueError(msg) + + # Get the file type + self.ftype = self._get_file_type() + self.log.debug(_("Input file {0} is of type {1}"). + format(path, self.ftype)) + + self.archive = archive + + self.tmpdir = None + + self.use_gi = use_gi + + tpl = self._translate("tosca", {}) + if tpl: + return self._write_output(tpl, output_dir) + + def _parse_args(self, raw_args=None): + parser = argparse.ArgumentParser( + description='RIFT TOSCA translator for descriptors') + + parser.add_argument( + "-f", + "--template-file", + required=True, + help="Template file to translate") + + parser.add_argument( + "-o", + "--output-dir", + help="Directory to output") + + parser.add_argument( + "-p", "--parameters", + help="Input parameters") + + parser.add_argument( + "-a", "--archive", + action="store_true", + help="Archive the translated files") + + parser.add_argument( + "--no-gi", + help="Do not use the YANG GI to generate descriptors", + action="store_true") + + parser.add_argument( + "--validate-only", + help="Validate template, no translation", + action="store_true") + + parser.add_argument( + "--debug", + help="Enable debug logging", + action="store_true") + + if raw_args: + args = parser.parse_args(raw_args) + else: + args = parser.parse_args() + return args + + def _parse_parameters(self, parameter_list): + parsed_inputs = {} + if parameter_list: + # Parameters are semi-colon separated + inputs = parameter_list.replace('"', '').split(';') + # Each parameter should be an assignment + for param in inputs: + keyvalue = param.split('=') + # Validate the parameter has both a name and value + msg = _("'%(param)s' is not a well-formed parameter.") % { + 'param': param} + if keyvalue.__len__() is 2: + # Assure parameter name is not zero-length or whitespace + stripped_name = keyvalue[0].strip() + if not stripped_name: + self.log.error(msg) + raise ValueError(msg) + # Add the valid parameter to the dictionary + parsed_inputs[keyvalue[0]] = keyvalue[1] + else: + self.log.error(msg) + raise ValueError(msg) + return parsed_inputs + + def get_entry_file(self): + # Extract the archive and get the entry file + if self.ftype == self.YAML: + return self.in_file + + self.prefix = '' + if self.ftype == self.ZIP: + self.tmpdir = tempfile.mkdtemp() + prevdir = os.getcwd() + try: + with zipfile.ZipFile(self.in_file) as zf: + self.prefix = os.path.commonprefix(zf.namelist()) + self.log.debug(_("Zipfile prefix is {0}"). + format(self.prefix)) + zf.extractall(self.tmpdir) + + # Set the execute bits on scripts as zipfile + # does not restore the permissions bits + os.chdir(self.tmpdir) + for fname in zf.namelist(): + if (fname.startswith('scripts/') and + os.path.isfile(fname)): + # Assume this is a script file + # Give all permissions to owner and read+execute + # for group and others + os.chmod(fname, + stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|stat.S_IROTH|stat.S_IXOTH) + + # TODO (pjoseph): Use the below code instead of extract all + # once unzip is installed on launchpad VMs + # zfile = os.path.abspath(self.in_file) + # os.chdir(self.tmpdir) + # zip_cmd = "unzip {}".format(zfile) + # subprocess.check_call(zip_cmd, + # #stdout=subprocess.PIPE, + # #stderr=subprocess.PIPE, + # shell=True,) + + except Exception as e: + msg = _("Exception extracting input file {0}: {1}"). \ + format(self.in_file, e) + self.log.error(msg) + self.log.exception(e) + os.chdir(prevdir) + shutil.rmtree(self.tmpdir) + self.tmpdir = None + raise ToscaEntryFileError(msg) + + os.chdir(self.tmpdir) + + try: + # Goto the TOSAC Metadata file + prefix_dir = os.path.join(self.tmpdir, self.prefix) + meta_file = os.path.join(prefix_dir, 'TOSCA-Metadata', + 'TOSCA.meta') + self.log.debug(_("Checking metadata file {0}").format(meta_file)) + if not os.path.exists(meta_file): + self.log.error(_("Not able to find metadata file in archive")) + return + + # Open the metadata file and get the entry file + with open(meta_file, 'r') as f: + meta = yaml.load(f) + + if 'Entry-Definitions' in meta: + entry_file = os.path.join(prefix_dir, + meta['Entry-Definitions']) + if os.path.exists(entry_file): + self.log.debug(_("TOSCA entry file is {0}"). + format(entry_file)) + return entry_file + + else: + msg = _("Unable to get the entry file: {0}"). \ + format(entry_file) + self.log.error(msg) + raise ToscaEntryFileNotFoundError(msg) + + else: + msg = _("Did not find entry definition " \ + "in metadata: {0}").format(meta) + self.log.error(msg) + raise ToscaNoEntryDefinitionError(msg) + + except Exception as e: + msg = _('Exception parsing metadata file {0}: {1}'). \ + format(meta_file, e) + self.log.error(msg) + self.log.exception(e) + raise ToscaEntryFileError(msg) + + finally: + os.chdir(prevdir) + + def _translate(self, sourcetype, parsed_params): + output = None + + # Check the input file + path = os.path.abspath(self.template_file) + self.in_file = path + a_file = os.path.isfile(path) + if not a_file: + msg = _("The path {} is not a valid file."). \ + format(self.template_file) + self.log.error(msg) + raise ValueError(msg) + + # Get the file type + self.ftype = self._get_file_type() + self.log.debug(_("Input file {0} is of type {1}"). + format(path, self.ftype)) + + if sourcetype == "tosca": + entry_file = self.get_entry_file() + if entry_file: + self.log.debug(_('Loading the tosca template.')) + tosca = ToscaTemplate(entry_file, parsed_params, True) + self.log.debug(_('TOSCA Template: {}').format(tosca.__dict__)) + translator = TOSCATranslator(self.log, tosca, parsed_params, + use_gi=self.use_gi) + self.log.debug(_('Translating the tosca template.')) + output = translator.translate() + return output + + def _copy_supporting_files(self, output_dir, files): + # Copy supporting files, if present in archive + if self.tmpdir: + # The files are refered relative to the definitions directory + arc_dir = os.path.join(self.tmpdir, + self.prefix, + 'Definitions') + prevdir = os.getcwd() + try: + os.chdir(arc_dir) + for fn in files: + fname = fn['name'] + fpath = os.path.abspath(fname) + ty = fn['type'] + if ty == 'image': + dest = os.path.join(output_dir, 'images') + elif ty == 'script': + dest = os.path.join(output_dir, 'scripts') + elif ty == 'cloud_init': + dest = os.path.join(output_dir, 'cloud_init') + else: + self.log.warn(_("Unknown file type {0} for {1}"). + format(ty, fname)) + continue + + self.log.debug(_("File type {0} copy from {1} to {2}"). + format(ty, fpath, dest)) + if os.path.exists(fpath): + # Copy the files to the appropriate dir + self.log.debug(_("Copy file(s) {0} to {1}"). + format(fpath, dest)) + if os.path.isdir(fpath): + # Copy directory structure like charm dir + shutil.copytree(fpath, dest) + else: + # Copy a single file + os.makedirs(dest, exist_ok=True) + shutil.copy2(fpath, dest) + + else: + self.log.warn(_("Could not find file {0} at {1}"). + format(fname, fpath)) + + except Exception as e: + self.log.error(_("Exception copying files {0}: {1}"). + format(arc_dir, e)) + self.log.exception(e) + + finally: + os.chdir(prevdir) + + def _create_checksum_file(self, output_dir): + # Create checkum for all files + flist = {} + for root, dirs, files in os.walk(output_dir): + rel_dir = root.replace(output_dir, '').lstrip('/') + + for f in files: + fpath = os.path.join(root, f) + # TODO (pjoseph): To be fixed when we can + # retrieve image files from Launchpad + if os.path.getsize(fpath) != 0: + flist[os.path.join(rel_dir, f)] = \ + ChecksumUtils.get_md5(fpath) + self.log.debug(_("Files in output_dir: {}").format(flist)) + + chksumfile = os.path.join(output_dir, 'checksums.txt') + with open(chksumfile, 'w') as c: + for key in sorted(flist.keys()): + c.write("{} {}\n".format(flist[key], key)) + + def _create_archive(self, desc_id, output_dir): + """Create a tar.gz archive for the descriptor""" + aname = desc_id + '.tar.gz' + apath = os.path.join(output_dir, aname) + self.log.debug(_("Generating archive: {}").format(apath)) + + prevdir = os.getcwd() + os.chdir(output_dir) + + # Generate the archive + tar_cmd = "tar zcvf {} {}".format(apath, desc_id) + self.log.debug(_("Generate archive: {}").format(tar_cmd)) + + try: + subprocess.check_call(tar_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True) + return apath + + except subprocess.CalledProcessError as e: + msg = _("Error creating archive with {}: {}"). \ + format(tar_cmd, e) + self.log.error(msg) + raise ToscaCreateArchiveError(msg) + + finally: + os.chdir(prevdir) + + def _write_output(self, output, output_dir=None): + out_files = [] + + if output_dir: + output_dir = os.path.abspath(output_dir) + + if output: + # Do the VNFDs first and then NSDs as later when + # loading in launchpad, VNFDs need to be loaded first + for key in [ManoTemplate.VNFD, ManoTemplate.NSD]: + for desc in output[key]: + if output_dir: + desc_id = desc[ManoTemplate.ID] + # Create separate directories for each descriptors + # Use the descriptor id to avoid name clash + subdir = os.path.join(output_dir, desc_id) + os.makedirs(subdir) + + output_file = os.path.join(subdir, + desc[ManoTemplate.NAME]+'.yml') + self.log.debug(_("Writing file {0}"). + format(output_file)) + with open(output_file, 'w+') as f: + f.write(desc[ManoTemplate.YANG]) + + if ManoTemplate.FILES in desc: + self._copy_supporting_files(subdir, + desc[ManoTemplate.FILES]) + + if self.archive: + # Create checksum file + self._create_checksum_file(subdir) + out_files.append(self._create_archive(desc_id, + output_dir)) + # Remove the desc directory + shutil.rmtree(subdir) + else: + print(_("Descriptor {0}:\n{1}"). + format(desc[ManoTemplate.NAME], + desc[ManoTemplate.YANG])) + + if output_dir and self.archive: + # Return the list of archive files + return out_files + + def _get_file_type(self): + m = magic.open(magic.MAGIC_MIME) + m.load() + typ = m.file(self.in_file) + if typ.startswith('text/plain'): + # Assume to be yaml + return self.YAML + elif typ.startswith('application/zip'): + return self.ZIP + else: + msg = _("The file {0} is not a supported type: {1}"). \ + format(self.in_file, typ) + self.log.error(msg) + raise ValueError(msg) + + +def main(args=None, log=None): + TranslatorShell(log=log).main(raw_args=args) + + +if __name__ == '__main__': + main()