common/python/rift/mano/tosca_translator/shell.py

   1 #
   2 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   3 # not use this file except in compliance with the License. You may obtain
   4 # a copy of the License at
   5 #
   6 # http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  10 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  11 # License for the specific language governing permissions and limitations
  12 # under the License.
  13
  14 # Copyright 2016 RIFT.io Inc
  15
  16
  17 import argparse
  18 import logging
  19 import logging.config
  20 import os
  21 import shutil
  22 import stat
  23 import subprocess
  24 import tempfile
  25 import zipfile
  26
  27 import magic
  28
  29 import yaml
  30
  31 from rift.mano.tosca_translator.common.utils import _
  32 from rift.mano.tosca_translator.common.utils import ChecksumUtils
  33 from rift.mano.tosca_translator.rwmano.syntax.mano_template import ManoTemplate
  34 from rift.mano.tosca_translator.rwmano.tosca_translator import TOSCATranslator
  35
  36 from toscaparser.tosca_template import ToscaTemplate
  37
  38
  39 """
  40 Test the tosca translation from command line as:
  41 #translator
  42   --template-file=<path to the YAML template or CSAR>
  43   --template-type=<type of template e.g. tosca>
  44   --parameters="purpose=test"
  45   --output_dir=<output directory>
  46   --archive
  47   --validate_only
  48 Takes following user arguments,
  49 . Path to the file that needs to be translated (required)
  50 . Input parameters (optional)
  51 . Write to output files in a dir (optional), else print on screen
  52 . Create archive or not
  53
  54 In order to use translator to only validate template,
  55 without actual translation, pass --validate-only along with
  56 other required arguments.
  57
  58 """
  59
  60
  61 class ToscaShellError(Exception):
  62     pass
  63
  64
  65 class ToscaEntryFileError(ToscaShellError):
  66     pass
  67
  68
  69 class ToscaNoEntryDefinitionError(ToscaShellError):
  70     pass
  71
  72
  73 class ToscaEntryFileNotFoundError(ToscaShellError):
  74     pass
  75
  76
  77 class ToscaCreateArchiveError(ToscaShellError):
  78     pass
  79
  80
  81 class TranslatorShell(object):
  82
  83     SUPPORTED_TYPES = ['tosca']
  84     COPY_DIRS = ['images']
  85     SUPPORTED_INPUTS = (YAML, ZIP) = ('yaml', 'zip')
  86
  87     def __init__(self, log=None):
  88         self.log = log
  89
  90     def main(self, raw_args=None):
  91         args = self._parse_args(raw_args)
  92
  93         if self.log is None:
  94             if args.debug:
  95                 logging.basicConfig(level=logging.DEBUG)
  96             else:
  97                 logging.basicConfig(level=logging.ERROR)
  98             self.log = logging.getLogger("tosca-translator")
  99
 100         self.template_file = args.template_file
 101
 102         parsed_params = {}
 103         if args.parameters:
 104             parsed_params = self._parse_parameters(args.parameters)
 105
 106         self.archive = False
 107         if args.archive:
 108             self.archive = True
 109
 110         self.tmpdir = None
 111
 112         if args.validate_only:
 113             a_file = os.path.isfile(args.template_file)
 114             tpl = ToscaTemplate(self.template_file, parsed_params, a_file)
 115             self.log.debug(_('Template = {}').format(tpl.__dict__))
 116             msg = (_('The input {} successfully passed ' \
 117                      'validation.').format(self.template_file))
 118             print(msg)
 119         else:
 120             self.use_gi = not args.no_gi
 121             tpl = self._translate("tosca", parsed_params)
 122             if tpl:
 123                 return self._write_output(tpl, args.output_dir)
 124
 125     def translate(self,
 126                   template_file,
 127                   output_dir=None,
 128                   use_gi=True,
 129                   archive=False,):
 130         self.template_file = template_file
 131
 132         # Check the input file
 133         path = os.path.abspath(template_file)
 134         self.in_file = path
 135         a_file = os.path.isfile(path)
 136         if not a_file:
 137             msg = _("The path {0} is not a valid file.").format(template_file)
 138             self.log.error(msg)
 139             raise ValueError(msg)
 140
 141         # Get the file type
 142         self.ftype = self._get_file_type()
 143         self.log.debug(_("Input file {0} is of type {1}").
 144                        format(path, self.ftype))
 145
 146         self.archive = archive
 147
 148         self.tmpdir = None
 149
 150         self.use_gi = use_gi
 151
 152         tpl = self._translate("tosca", {})
 153         if tpl:
 154             return self._write_output(tpl, output_dir)
 155
 156     def _parse_args(self, raw_args=None):
 157         parser = argparse.ArgumentParser(
 158             description='RIFT TOSCA translator for descriptors')
 159
 160         parser.add_argument(
 161             "-f",
 162             "--template-file",
 163             required=True,
 164             help="Template file to translate")
 165
 166         parser.add_argument(
 167             "-o",
 168             "--output-dir",
 169             help="Directory to output")
 170
 171         parser.add_argument(
 172             "-p", "--parameters",
 173             help="Input parameters")
 174
 175         parser.add_argument(
 176             "-a", "--archive",
 177             action="store_true",
 178             help="Archive the translated files")
 179
 180         parser.add_argument(
 181             "--no-gi",
 182             help="Do not use the YANG GI to generate descriptors",
 183             action="store_true")
 184
 185         parser.add_argument(
 186             "--validate-only",
 187             help="Validate template, no translation",
 188             action="store_true")
 189
 190         parser.add_argument(
 191             "--debug",
 192             help="Enable debug logging",
 193             action="store_true")
 194
 195         if raw_args:
 196             args = parser.parse_args(raw_args)
 197         else:
 198             args = parser.parse_args()
 199         return args
 200
 201     def _parse_parameters(self, parameter_list):
 202         parsed_inputs = {}
 203         if parameter_list:
 204             # Parameters are semi-colon separated
 205             inputs = parameter_list.replace('"', '').split(';')
 206             # Each parameter should be an assignment
 207             for param in inputs:
 208                 keyvalue = param.split('=')
 209                 # Validate the parameter has both a name and value
 210                 msg = _("'%(param)s' is not a well-formed parameter.") % {
 211                     'param': param}
 212                 if keyvalue.__len__() is 2:
 213                     # Assure parameter name is not zero-length or whitespace
 214                     stripped_name = keyvalue[0].strip()
 215                     if not stripped_name:
 216                         self.log.error(msg)
 217                         raise ValueError(msg)
 218                     # Add the valid parameter to the dictionary
 219                     parsed_inputs[keyvalue[0]] = keyvalue[1]
 220                 else:
 221                     self.log.error(msg)
 222                     raise ValueError(msg)
 223         return parsed_inputs
 224
 225     def get_entry_file(self):
 226         # Extract the archive and get the entry file
 227         if self.ftype == self.YAML:
 228             return self.in_file
 229
 230         self.prefix = ''
 231         if self.ftype == self.ZIP:
 232             self.tmpdir = tempfile.mkdtemp()
 233             prevdir = os.getcwd()
 234             try:
 235                 with zipfile.ZipFile(self.in_file) as zf:
 236                     self.prefix = os.path.commonprefix(zf.namelist())
 237                     self.log.debug(_("Zipfile prefix is {0}").
 238                                    format(self.prefix))
 239                     zf.extractall(self.tmpdir)
 240
 241                     # Set the execute bits on scripts as zipfile
 242                     # does not restore the permissions bits
 243                     os.chdir(self.tmpdir)
 244                     for fname in zf.namelist():
 245                         if (fname.startswith('scripts/') and
 246                             os.path.isfile(fname)):
 247                             # Assume this is a script file
 248                             # Give all permissions to owner and read+execute
 249                             # for group and others
 250                             os.chmod(fname,
 251                                      stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|stat.S_IROTH|stat.S_IXOTH)
 252
 253                 # TODO (pjoseph): Use the below code instead of extract all
 254                 # once unzip is installed on launchpad VMs
 255                 # zfile = os.path.abspath(self.in_file)
 256                 # os.chdir(self.tmpdir)
 257                 # zip_cmd = "unzip {}".format(zfile)
 258                 # subprocess.check_call(zip_cmd,
 259                 #                       #stdout=subprocess.PIPE,
 260                 #                       #stderr=subprocess.PIPE,
 261                 #                       shell=True,)
 262
 263             except Exception as e:
 264                 msg = _("Exception extracting input file {0}: {1}"). \
 265                       format(self.in_file, e)
 266                 self.log.error(msg)
 267                 self.log.exception(e)
 268                 os.chdir(prevdir)
 269                 shutil.rmtree(self.tmpdir)
 270                 self.tmpdir = None
 271                 raise ToscaEntryFileError(msg)
 272
 273         os.chdir(self.tmpdir)
 274
 275         try:
 276             # Goto the TOSAC Metadata file
 277             prefix_dir = os.path.join(self.tmpdir, self.prefix)
 278             meta_file = os.path.join(prefix_dir, 'TOSCA-Metadata',
 279                                      'TOSCA.meta')
 280             self.log.debug(_("Checking metadata file {0}").format(meta_file))
 281             if not os.path.exists(meta_file):
 282                 self.log.error(_("Not able to find metadata file in archive"))
 283                 return
 284
 285             # Open the metadata file and get the entry file
 286             with open(meta_file, 'r') as f:
 287                 meta = yaml.load(f)
 288
 289                 if 'Entry-Definitions' in meta:
 290                     entry_file = os.path.join(prefix_dir,
 291                                               meta['Entry-Definitions'])
 292                     if os.path.exists(entry_file):
 293                         self.log.debug(_("TOSCA entry file is {0}").
 294                                        format(entry_file))
 295                         return entry_file
 296
 297                     else:
 298                         msg = _("Unable to get the entry file: {0}"). \
 299                               format(entry_file)
 300                         self.log.error(msg)
 301                         raise ToscaEntryFileNotFoundError(msg)
 302
 303                 else:
 304                     msg = _("Did not find entry definition " \
 305                             "in metadata: {0}").format(meta)
 306                     self.log.error(msg)
 307                     raise ToscaNoEntryDefinitionError(msg)
 308
 309         except Exception as e:
 310             msg = _('Exception parsing metadata file {0}: {1}'). \
 311                   format(meta_file, e)
 312             self.log.error(msg)
 313             self.log.exception(e)
 314             raise ToscaEntryFileError(msg)
 315
 316         finally:
 317             os.chdir(prevdir)
 318
 319     def _translate(self, sourcetype, parsed_params):
 320         output = None
 321
 322         # Check the input file
 323         path = os.path.abspath(self.template_file)
 324         self.in_file = path
 325         a_file = os.path.isfile(path)
 326         if not a_file:
 327             msg = _("The path {} is not a valid file."). \
 328                   format(self.template_file)
 329             self.log.error(msg)
 330             raise ValueError(msg)
 331
 332         # Get the file type
 333         self.ftype = self._get_file_type()
 334         self.log.debug(_("Input file {0} is of type {1}").
 335                        format(path, self.ftype))
 336
 337         if sourcetype == "tosca":
 338             entry_file = self.get_entry_file()
 339             if entry_file:
 340                 self.log.debug(_('Loading the tosca template.'))
 341                 tosca = ToscaTemplate(entry_file, parsed_params, True)
 342                 self.log.debug(_('TOSCA Template: {}').format(tosca.__dict__))
 343                 translator = TOSCATranslator(self.log, tosca, parsed_params,
 344                                          use_gi=self.use_gi)
 345                 self.log.debug(_('Translating the tosca template.'))
 346                 output = translator.translate()
 347         return output
 348
 349     def _copy_supporting_files(self, output_dir, files):
 350         # Copy supporting files, if present in archive
 351         if self.tmpdir:
 352             # The files are refered relative to the definitions directory
 353             arc_dir = os.path.join(self.tmpdir,
 354                                    self.prefix,
 355                                    'Definitions')
 356             prevdir = os.getcwd()
 357             try:
 358                 os.chdir(arc_dir)
 359                 for fn in files:
 360                     fname = fn['name']
 361                     fpath = os.path.abspath(fname)
 362                     ty = fn['type']
 363                     if ty == 'image':
 364                         dest = os.path.join(output_dir, 'images')
 365                     elif ty == 'script':
 366                         dest = os.path.join(output_dir, 'scripts')
 367                     elif ty == 'cloud_init':
 368                         dest = os.path.join(output_dir, 'cloud_init')
 369                     else:
 370                         self.log.warn(_("Unknown file type {0} for {1}").
 371                                       format(ty, fname))
 372                         continue
 373
 374                     self.log.debug(_("File type {0} copy from {1} to {2}").
 375                                    format(ty, fpath, dest))
 376                     if os.path.exists(fpath):
 377                         # Copy the files to the appropriate dir
 378                         self.log.debug(_("Copy file(s) {0} to {1}").
 379                                          format(fpath, dest))
 380                         if os.path.isdir(fpath):
 381                             # Copy directory structure like charm dir
 382                             shutil.copytree(fpath, dest)
 383                         else:
 384                             # Copy a single file
 385                             os.makedirs(dest, exist_ok=True)
 386                             shutil.copy2(fpath, dest)
 387
 388                     else:
 389                         self.log.warn(_("Could not find file {0} at {1}").
 390                                       format(fname, fpath))
 391
 392             except Exception as e:
 393                 self.log.error(_("Exception copying files {0}: {1}").
 394                                format(arc_dir, e))
 395                 self.log.exception(e)
 396
 397             finally:
 398                 os.chdir(prevdir)
 399
 400     def _create_checksum_file(self, output_dir):
 401         # Create checkum for all files
 402         flist = {}
 403         for root, dirs, files in os.walk(output_dir):
 404             rel_dir = root.replace(output_dir, '').lstrip('/')
 405
 406             for f in files:
 407                 fpath = os.path.join(root, f)
 408                 # TODO (pjoseph): To be fixed when we can
 409                 # retrieve image files from Launchpad
 410                 if os.path.getsize(fpath) != 0:
 411                     flist[os.path.join(rel_dir, f)] = \
 412                                                 ChecksumUtils.get_md5(fpath)
 413                     self.log.debug(_("Files in output_dir: {}").format(flist))
 414
 415                 chksumfile = os.path.join(output_dir, 'checksums.txt')
 416                 with open(chksumfile, 'w') as c:
 417                     for key in sorted(flist.keys()):
 418                         c.write("{}  {}\n".format(flist[key], key))
 419
 420     def _create_archive(self, desc_id, output_dir):
 421         """Create a tar.gz archive for the descriptor"""
 422         aname = desc_id + '.tar.gz'
 423         apath = os.path.join(output_dir, aname)
 424         self.log.debug(_("Generating archive: {}").format(apath))
 425
 426         prevdir = os.getcwd()
 427         os.chdir(output_dir)
 428
 429         # Generate the archive
 430         tar_cmd = "tar zcvf {} {}".format(apath, desc_id)
 431         self.log.debug(_("Generate archive: {}").format(tar_cmd))
 432
 433         try:
 434             subprocess.check_call(tar_cmd,
 435                                   stdout=subprocess.PIPE,
 436                                   stderr=subprocess.PIPE,
 437                                   shell=True)
 438             return apath
 439
 440         except subprocess.CalledProcessError as e:
 441             msg = _("Error creating archive with {}: {}"). \
 442                            format(tar_cmd, e)
 443             self.log.error(msg)
 444             raise ToscaCreateArchiveError(msg)
 445
 446         finally:
 447             os.chdir(prevdir)
 448
 449     def _write_output(self, output, output_dir=None):
 450         out_files = []
 451
 452         if output_dir:
 453             output_dir = os.path.abspath(output_dir)
 454
 455         if output:
 456             # Do the VNFDs first and then NSDs as later when
 457             # loading in launchpad, VNFDs need to be loaded first
 458             for key in [ManoTemplate.VNFD, ManoTemplate.NSD]:
 459                 for desc in output[key]:
 460                     if output_dir:
 461                         desc_id = desc[ManoTemplate.ID]
 462                         # Create separate directories for each descriptors
 463                         # Use the descriptor id to avoid name clash
 464                         subdir = os.path.join(output_dir, desc_id)
 465                         os.makedirs(subdir)
 466
 467                         output_file = os.path.join(subdir,
 468                                             desc[ManoTemplate.NAME]+'.yml')
 469                         self.log.debug(_("Writing file {0}").
 470                                        format(output_file))
 471                         with open(output_file, 'w+') as f:
 472                             f.write(desc[ManoTemplate.YANG])
 473
 474                         if ManoTemplate.FILES in desc:
 475                             self._copy_supporting_files(subdir,
 476                                                 desc[ManoTemplate.FILES])
 477
 478                         if self.archive:
 479                             # Create checksum file
 480                             self._create_checksum_file(subdir)
 481                             out_files.append(self._create_archive(desc_id,
 482                                                                   output_dir))
 483                             # Remove the desc directory
 484                             shutil.rmtree(subdir)
 485                     else:
 486                         print(_("Descriptor {0}:\n{1}").
 487                               format(desc[ManoTemplate.NAME],
 488                                      desc[ManoTemplate.YANG]))
 489
 490             if output_dir and self.archive:
 491                 # Return the list of archive files
 492                 return out_files
 493
 494     def _get_file_type(self):
 495         m = magic.open(magic.MAGIC_MIME)
 496         m.load()
 497         typ = m.file(self.in_file)
 498         if typ.startswith('text/plain'):
 499             # Assume to be yaml
 500             return self.YAML
 501         elif typ.startswith('application/zip'):
 502             return self.ZIP
 503         else:
 504             msg = _("The file {0} is not a supported type: {1}"). \
 505                   format(self.in_file, typ)
 506             self.log.error(msg)
 507             raise ValueError(msg)
 508
 509
 510 def main(args=None, log=None):
 511     TranslatorShell(log=log).main(raw_args=args)
 512
 513
 514 if __name__ == '__main__':
 515     main()