common/python/rift/mano/tosca_translator/shell.py

   1 #
   2 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   3 # not use this file except in compliance with the License. You may obtain
   4 # a copy of the License at
   5 #
   6 # http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  10 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  11 # License for the specific language governing permissions and limitations
  12 # under the License.
  13
  14 # Copyright 2016 RIFT.io Inc
  15
  16
  17 import argparse
  18 import logging
  19 import logging.config
  20 import os
  21 import shutil
  22 import stat
  23 import subprocess
  24 import tempfile
  25 import zipfile
  26
  27 import magic
  28
  29 import yaml
  30
  31 from rift.mano.tosca_translator.common.utils import _
  32 from rift.mano.tosca_translator.common.utils import ChecksumUtils
  33 from rift.mano.tosca_translator.rwmano.syntax.mano_template import ManoTemplate
  34 from rift.mano.tosca_translator.rwmano.tosca_translator import TOSCATranslator
  35
  36 from toscaparser.tosca_template import ToscaTemplate
  37
  38
  39 """
  40 Test the tosca translation from command line as:
  41 #translator
  42   --template-file=<path to the YAML template or CSAR>
  43   --template-type=<type of template e.g. tosca>
  44   --parameters="purpose=test"
  45   --output_dir=<output directory>
  46   --archive
  47   --validate_only
  48 Takes following user arguments,
  49 . Path to the file that needs to be translated (required)
  50 . Input parameters (optional)
  51 . Write to output files in a dir (optional), else print on screen
  52 . Create archive or not
  53
  54 In order to use translator to only validate template,
  55 without actual translation, pass --validate-only along with
  56 other required arguments.
  57
  58 """
  59
  60
  61 class ToscaShellError(Exception):
  62     pass
  63
  64
  65 class ToscaEntryFileError(ToscaShellError):
  66     pass
  67
  68
  69 class ToscaNoEntryDefinitionError(ToscaShellError):
  70     pass
  71
  72
  73 class ToscaEntryFileNotFoundError(ToscaShellError):
  74     pass
  75
  76
  77 class ToscaCreateArchiveError(ToscaShellError):
  78     pass
  79
  80
  81 class TranslatorShell(object):
  82
  83     SUPPORTED_TYPES = ['tosca']
  84     COPY_DIRS = ['images']
  85     SUPPORTED_INPUTS = (YAML, ZIP) = ('yaml', 'zip')
  86
  87     def __init__(self, log=None):
  88         self.log = log
  89
  90     def main(self, raw_args=None):
  91         args = self._parse_args(raw_args)
  92
  93         if self.log is None:
  94             if args.debug:
  95                 logging.basicConfig(level=logging.DEBUG)
  96             else:
  97                 logging.basicConfig(level=logging.ERROR)
  98             self.log = logging.getLogger("tosca-translator")
  99
 100         self.template_file = args.template_file
 101
 102         parsed_params = {}
 103         if args.parameters:
 104             parsed_params = self._parse_parameters(args.parameters)
 105
 106         self.archive = False
 107         if args.archive:
 108             self.archive = True
 109
 110         self.tmpdir = None
 111
 112         if args.validate_only:
 113             a_file = os.path.isfile(args.template_file)
 114             tpl = ToscaTemplate(self.template_file, parsed_params, a_file)
 115             self.log.debug(_('Template = {}').format(tpl.__dict__))
 116             msg = (_('The input {} successfully passed ' \
 117                      'validation.').format(self.template_file))
 118             print(msg)
 119         else:
 120             self.use_gi = not args.no_gi
 121             tpl = self._translate("tosca", parsed_params)
 122             if tpl:
 123                 return self._write_output(tpl, args.output_dir)
 124
 125     def translate(self,
 126                   template_file,
 127                   output_dir=None,
 128                   use_gi=True,
 129                   archive=False,):
 130         self.template_file = template_file
 131
 132         # Check the input file
 133         path = os.path.abspath(template_file)
 134         self.in_file = path
 135         a_file = os.path.isfile(path)
 136         if not a_file:
 137             msg = _("The path {0} is not a valid file.").format(template_file)
 138             self.log.error(msg)
 139             raise ValueError(msg)
 140
 141         # Get the file type
 142         self.ftype = self._get_file_type()
 143         self.log.debug(_("Input file {0} is of type {1}").
 144                        format(path, self.ftype))
 145
 146         self.archive = archive
 147
 148         self.tmpdir = None
 149
 150         self.use_gi = use_gi
 151
 152         tpl = self._translate("tosca", {})
 153         if tpl:
 154             return self._write_output(tpl, output_dir)
 155
 156     def _parse_args(self, raw_args=None):
 157         parser = argparse.ArgumentParser(
 158             description='RIFT TOSCA translator for descriptors')
 159
 160         parser.add_argument(
 161             "-f",
 162             "--template-file",
 163             required=True,
 164             help="Template file to translate")
 165
 166         parser.add_argument(
 167             "-o",
 168             "--output-dir",
 169             help="Directory to output")
 170
 171         parser.add_argument(
 172             "-p", "--parameters",
 173             help="Input parameters")
 174
 175         parser.add_argument(
 176             "-a", "--archive",
 177             action="store_true",
 178             help="Archive the translated files")
 179
 180         parser.add_argument(
 181             "--no-gi",
 182             help="Do not use the YANG GI to generate descriptors",
 183             action="store_true")
 184
 185         parser.add_argument(
 186             "--validate-only",
 187             help="Validate template, no translation",
 188             action="store_true")
 189
 190         parser.add_argument(
 191             "--debug",
 192             help="Enable debug logging",
 193             action="store_true")
 194
 195         if raw_args:
 196             args = parser.parse_args(raw_args)
 197         else:
 198             args = parser.parse_args()
 199         return args
 200
 201     def _parse_parameters(self, parameter_list):
 202         parsed_inputs = {}
 203         if parameter_list:
 204             # Parameters are semi-colon separated
 205             inputs = parameter_list.replace('"', '').split(';')
 206             # Each parameter should be an assignment
 207             for param in inputs:
 208                 keyvalue = param.split('=')
 209                 # Validate the parameter has both a name and value
 210                 msg = _("'%(param)s' is not a well-formed parameter.") % {
 211                     'param': param}
 212                 if keyvalue.__len__() is 2:
 213                     # Assure parameter name is not zero-length or whitespace
 214                     stripped_name = keyvalue[0].strip()
 215                     if not stripped_name:
 216                         self.log.error(msg)
 217                         raise ValueError(msg)
 218                     # Add the valid parameter to the dictionary
 219                     parsed_inputs[keyvalue[0]] = keyvalue[1]
 220                 else:
 221                     self.log.error(msg)
 222                     raise ValueError(msg)
 223         return parsed_inputs
 224
 225     def get_entry_file(self):
 226         # Extract the archive and get the entry file
 227         if self.ftype == self.YAML:
 228             return self.in_file
 229
 230         self.prefix = ''
 231         if self.ftype == self.ZIP:
 232             self.tmpdir = tempfile.mkdtemp()
 233             prevdir = os.getcwd()
 234             try:
 235                 with zipfile.ZipFile(self.in_file) as zf:
 236                     self.prefix = os.path.commonprefix(zf.namelist())
 237                     self.log.debug(_("Zipfile prefix is {0}").
 238                                    format(self.prefix))
 239                     zf.extractall(self.tmpdir)
 240
 241                     # Set the execute bits on scripts as zipfile
 242                     # does not restore the permissions bits
 243                     os.chdir(self.tmpdir)
 244                     for fname in zf.namelist():
 245                         if (fname.startswith('scripts/') and
 246                             os.path.isfile(fname)):
 247                             # Assume this is a script file
 248                             # Give all permissions to owner and read+execute
 249                             # for group and others
 250                             os.chmod(fname,
 251                                      stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|stat.S_IROTH|stat.S_IXOTH)
 252
 253                 # TODO (pjoseph): Use the below code instead of extract all
 254                 # once unzip is installed on launchpad VMs
 255                 # zfile = os.path.abspath(self.in_file)
 256                 # os.chdir(self.tmpdir)
 257                 # zip_cmd = "unzip {}".format(zfile)
 258                 # subprocess.check_call(zip_cmd,
 259                 #                       #stdout=subprocess.PIPE,
 260                 #                       #stderr=subprocess.PIPE,
 261                 #                       shell=True,)
 262
 263             except Exception as e:
 264                 msg = _("Exception extracting input file {0}: {1}"). \
 265                       format(self.in_file, e)
 266                 self.log.error(msg)
 267                 self.log.exception(e)
 268                 os.chdir(prevdir)
 269                 shutil.rmtree(self.tmpdir)
 270                 self.tmpdir = None
 271                 raise ToscaEntryFileError(msg)
 272
 273         os.chdir(self.tmpdir)
 274
 275         try:
 276             # Goto the TOSAC Metadata file
 277             prefix_dir = os.path.join(self.tmpdir, self.prefix)
 278             meta_file = os.path.join(prefix_dir, 'TOSCA-Metadata',
 279                                      'TOSCA.meta')
 280             self.log.debug(_("Checking metadata file {0}").format(meta_file))
 281             if not os.path.exists(meta_file):
 282                 self.log.error(_("Not able to find metadata file in archive"))
 283                 return
 284
 285             # Open the metadata file and get the entry file
 286             with open(meta_file, 'r') as f:
 287                 meta = yaml.load(f)
 288
 289                 if 'Entry-Definitions' in meta:
 290                     entry_file = os.path.join(prefix_dir,
 291                                               meta['Entry-Definitions'])
 292                     if os.path.exists(entry_file):
 293                         self.log.debug(_("TOSCA entry file is {0}").
 294                                        format(entry_file))
 295                         return entry_file
 296
 297                     else:
 298                         msg = _("Unable to get the entry file: {0}"). \
 299                               format(entry_file)
 300                         self.log.error(msg)
 301                         raise ToscaEntryFileNotFoundError(msg)
 302
 303                 else:
 304                     msg = _("Did not find entry definition " \
 305                             "in metadata: {0}").format(meta)
 306                     self.log.error(msg)
 307                     raise ToscaNoEntryDefinitionError(msg)
 308
 309         except Exception as e:
 310             msg = _('Exception parsing metadata file {0}: {1}'). \
 311                   format(meta_file, e)
 312             self.log.error(msg)
 313             self.log.exception(e)
 314             raise ToscaEntryFileError(msg)
 315
 316         finally:
 317             os.chdir(prevdir)
 318
 319     def _translate(self, sourcetype, parsed_params):
 320         output = None
 321
 322         # Check the input file
 323         path = os.path.abspath(self.template_file)
 324         self.in_file = path
 325         a_file = os.path.isfile(path)
 326         if not a_file:
 327             msg = _("The path {} is not a valid file."). \
 328                   format(self.template_file)
 329             self.log.error(msg)
 330             raise ValueError(msg)
 331
 332         # Get the file type
 333         self.ftype = self._get_file_type()
 334         self.log.debug(_("Input file {0} is of type {1}").
 335                        format(path, self.ftype))
 336
 337         if sourcetype == "tosca":
 338             entry_file = self.get_entry_file()
 339             if entry_file:
 340                 self.log.debug(_('Loading the tosca template.'))
 341                 tosca = ToscaTemplate(entry_file, parsed_params, True)
 342                 self.log.debug(_('TOSCA Template: {}').format(tosca.__dict__))
 343                 translator = TOSCATranslator(self.log, tosca, parsed_params,
 344                                          use_gi=self.use_gi)
 345                 self.log.debug(_('Translating the tosca template.'))
 346                 output = translator.translate()
 347         return output
 348
 349     def _copy_supporting_files(self, output_dir, files):
 350         # Copy supporting files, if present in archive
 351         if self.tmpdir:
 352             # The files are refered relative to the definitions directory
 353             arc_dir = os.path.join(self.tmpdir,
 354                                    self.prefix,
 355                                    'Definitions')
 356             prevdir = os.getcwd()
 357             try:
 358                 os.chdir(arc_dir)
 359                 for fn in files:
 360                     fname = fn['name']
 361                     fpath = os.path.abspath(fname)
 362                     ty = fn['type']
 363                     if ty == 'image':
 364                         dest = os.path.join(output_dir, 'images')
 365                     elif ty == 'script':
 366                         dest = os.path.join(output_dir, 'scripts')
 367                     elif ty == 'icons':
 368                         dest = os.path.join(output_dir, 'icons')
 369                     elif ty == 'cloud_init':
 370                         dest = os.path.join(output_dir, 'cloud_init')
 371                     else:
 372                         self.log.warn(_("Unknown file type {0} for {1}").
 373                                       format(ty, fname))
 374                         continue
 375
 376                     self.log.debug(_("File type {0} copy from {1} to {2}").
 377                                    format(ty, fpath, dest))
 378                     if os.path.exists(fpath):
 379                         # Copy the files to the appropriate dir
 380                         self.log.debug(_("Copy file(s) {0} to {1}").
 381                                          format(fpath, dest))
 382                         if os.path.isdir(fpath):
 383                             # Copy directory structure like charm dir
 384                             shutil.copytree(fpath, dest)
 385                         else:
 386                             # Copy a single file
 387                             os.makedirs(dest, exist_ok=True)
 388                             shutil.copy2(fpath, dest)
 389
 390                     else:
 391                         self.log.warn(_("Could not find file {0} at {1}").
 392                                       format(fname, fpath))
 393
 394             except Exception as e:
 395                 self.log.error(_("Exception copying files {0}: {1}").
 396                                format(arc_dir, e))
 397                 self.log.exception(e)
 398
 399             finally:
 400                 os.chdir(prevdir)
 401
 402     def _create_checksum_file(self, output_dir):
 403         # Create checkum for all files
 404         flist = {}
 405         for root, dirs, files in os.walk(output_dir):
 406             rel_dir = root.replace(output_dir, '').lstrip('/')
 407
 408             for f in files:
 409                 fpath = os.path.join(root, f)
 410                 # TODO (pjoseph): To be fixed when we can
 411                 # retrieve image files from Launchpad
 412                 if os.path.getsize(fpath) != 0:
 413                     flist[os.path.join(rel_dir, f)] = \
 414                                                 ChecksumUtils.get_md5(fpath)
 415                     self.log.debug(_("Files in output_dir: {}").format(flist))
 416
 417                 chksumfile = os.path.join(output_dir, 'checksums.txt')
 418                 with open(chksumfile, 'w') as c:
 419                     for key in sorted(flist.keys()):
 420                         c.write("{}  {}\n".format(flist[key], key))
 421
 422     def _create_archive(self, desc_id, output_dir):
 423         """Create a tar.gz archive for the descriptor"""
 424         aname = desc_id + '.tar.gz'
 425         apath = os.path.join(output_dir, aname)
 426         self.log.debug(_("Generating archive: {}").format(apath))
 427
 428         prevdir = os.getcwd()
 429         os.chdir(output_dir)
 430
 431         # Generate the archive
 432         tar_cmd = "tar zcvf {} {}".format(apath, desc_id)
 433         self.log.debug(_("Generate archive: {}").format(tar_cmd))
 434
 435         try:
 436             subprocess.check_call(tar_cmd,
 437                                   stdout=subprocess.PIPE,
 438                                   stderr=subprocess.PIPE,
 439                                   shell=True)
 440             return apath
 441
 442         except subprocess.CalledProcessError as e:
 443             msg = _("Error creating archive with {}: {}"). \
 444                            format(tar_cmd, e)
 445             self.log.error(msg)
 446             raise ToscaCreateArchiveError(msg)
 447
 448         finally:
 449             os.chdir(prevdir)
 450
 451     def _write_output(self, output, output_dir=None):
 452         out_files = []
 453
 454         if output_dir:
 455             output_dir = os.path.abspath(output_dir)
 456
 457         if output:
 458             # Do the VNFDs first and then NSDs as later when
 459             # loading in launchpad, VNFDs need to be loaded first
 460             for key in [ManoTemplate.VNFD, ManoTemplate.NSD]:
 461                 for desc in output[key]:
 462                     if output_dir:
 463                         desc_id = desc[ManoTemplate.ID]
 464                         # Create separate directories for each descriptors
 465                         # Use the descriptor id to avoid name clash
 466                         subdir = os.path.join(output_dir, desc_id)
 467                         os.makedirs(subdir)
 468
 469                         output_file = os.path.join(subdir,
 470                                             desc[ManoTemplate.NAME]+'.yml')
 471                         self.log.debug(_("Writing file {0}").
 472                                        format(output_file))
 473                         with open(output_file, 'w+') as f:
 474                             f.write(desc[ManoTemplate.YANG])
 475
 476                         if ManoTemplate.FILES in desc:
 477                             self._copy_supporting_files(subdir,
 478                                                 desc[ManoTemplate.FILES])
 479
 480                         if self.archive:
 481                             # Create checksum file
 482                             self._create_checksum_file(subdir)
 483                             out_files.append(self._create_archive(desc_id,
 484                                                                   output_dir))
 485                             # Remove the desc directory
 486                             shutil.rmtree(subdir)
 487                     else:
 488                         print(_("Descriptor {0}:\n{1}").
 489                               format(desc[ManoTemplate.NAME],
 490                                      desc[ManoTemplate.YANG]))
 491
 492             if output_dir and self.archive:
 493                 # Return the list of archive files
 494                 return out_files
 495
 496     def _get_file_type(self):
 497         m = magic.open(magic.MAGIC_MIME)
 498         m.load()
 499         typ = m.file(self.in_file)
 500         if typ.startswith('text/plain'):
 501             # Assume to be yaml
 502             return self.YAML
 503         elif typ.startswith('application/zip'):
 504             return self.ZIP
 505         else:
 506             msg = _("The file {0} is not a supported type: {1}"). \
 507                   format(self.in_file, typ)
 508             self.log.error(msg)
 509             raise ValueError(msg)
 510
 511
 512 def main(args=None, log=None):
 513     TranslatorShell(log=log).main(raw_args=args)
 514
 515
 516 if __name__ == '__main__':
 517     main()