RO/osm_ro/http_tools/request_processing.py

   1 # -*- coding: utf-8 -*-
   2 ##
   3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   4 # not use this file except in compliance with the License. You may obtain
   5 # a copy of the License at
   6 #
   7 #         http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  12 # License for the specific language governing permissions and limitations
  13 # under the License.
  14 ##
  15
  16 #
  17 # Util functions previously in `httpserver`
  18 #
  19
  20 __author__ = "Alfonso Tierno, Gerardo Garcia"
  21
  22 import json
  23 import logging
  24
  25 import bottle
  26 import yaml
  27 from jsonschema import exceptions as js_e
  28 from jsonschema import validate as js_v
  29
  30 from . import errors as httperrors
  31 from io import TextIOWrapper
  32
  33 logger = logging.getLogger('openmano.http')
  34
  35
  36 def remove_clear_passwd(data):
  37     """
  38     Removes clear passwords from the data received
  39     :param data: data with clear password
  40     :return: data without the password information
  41     """
  42
  43     passw = ['password: ', 'passwd: ']
  44
  45     for pattern in passw:
  46         init = data.find(pattern)
  47         while init != -1:
  48             end = data.find('\n', init)
  49             data = data[:init] + '{}******'.format(pattern) + data[end:]
  50             init += 1
  51             init = data.find(pattern, init)
  52     return data
  53
  54
  55 def change_keys_http2db(data, http_db, reverse=False):
  56     '''Change keys of dictionary data acording to the key_dict values
  57     This allow change from http interface names to database names.
  58     When reverse is True, the change is otherwise
  59     Attributes:
  60         data: can be a dictionary or a list
  61         http_db: is a dictionary with hhtp names as keys and database names as value
  62         reverse: by default change is done from http api to database.
  63             If True change is done otherwise.
  64     Return: None, but data is modified'''
  65     if type(data) is tuple or type(data) is list:
  66         for d in data:
  67             change_keys_http2db(d, http_db, reverse)
  68     elif type(data) is dict or type(data) is bottle.FormsDict:
  69         if reverse:
  70             for k,v in http_db.items():
  71                 if v in data: data[k]=data.pop(v)
  72         else:
  73             for k,v in http_db.items():
  74                 if k in data: data[v]=data.pop(k)
  75
  76
  77 def format_out(data):
  78     '''Return string of dictionary data according to requested json, yaml, xml.
  79     By default json
  80     '''
  81     logger.debug("OUT: " + yaml.safe_dump(data, explicit_start=True, indent=4, default_flow_style=False, tags=False, allow_unicode=True) )
  82     accept = bottle.request.headers.get('Accept')
  83     if accept and 'application/yaml' in accept:
  84         bottle.response.content_type='application/yaml'
  85         return yaml.safe_dump(
  86                 data, explicit_start=True, indent=4, default_flow_style=False,
  87                 tags=False, allow_unicode=True) #, canonical=True, default_style='"'
  88     else: #by default json
  89         bottle.response.content_type='application/json'
  90         #return data #json no style
  91         return json.dumps(data, indent=4) + "\n"
  92
  93
  94 def format_in(default_schema, version_fields=None, version_dict_schema=None, confidential_data=False):
  95     """
  96     Parse the content of HTTP request against a json_schema
  97
  98     :param default_schema: The schema to be parsed by default
  99         if no version field is found in the client data.
 100         In None no validation is done
 101     :param version_fields: If provided it contains a tuple or list with the
 102         fields to iterate across the client data to obtain the version
 103     :param version_dict_schema: It contains a dictionary with the version as key,
 104         and json schema to apply as value.
 105         It can contain a None as key, and this is apply
 106         if the client data version does not match any key
 107     :return:  user_data, used_schema: if the data is successfully decoded and
 108         matches the schema.
 109
 110     Launch a bottle abort if fails
 111     """
 112     #print "HEADERS :" + str(bottle.request.headers.items())
 113     try:
 114         error_text = "Invalid header format "
 115         format_type = bottle.request.headers.get('Content-Type', 'application/json')
 116         if 'application/json' in format_type:
 117             error_text = "Invalid json format "
 118             #Use the json decoder instead of bottle decoder because it informs about the location of error formats with a ValueError exception
 119             client_data = json.load(TextIOWrapper(bottle.request.body, encoding="utf-8"))  # TODO py3
 120             #client_data = bottle.request.json()
 121         elif 'application/yaml' in format_type:
 122             error_text = "Invalid yaml format "
 123             client_data = yaml.load(bottle.request.body, Loader=yaml.Loader)
 124         elif 'application/xml' in format_type:
 125             bottle.abort(501, "Content-Type: application/xml not supported yet.")
 126         else:
 127             logger.warning('Content-Type ' + str(format_type) + ' not supported.')
 128             bottle.abort(httperrors.Not_Acceptable, 'Content-Type ' + str(format_type) + ' not supported.')
 129             return
 130         # if client_data == None:
 131         #    bottle.abort(httperrors.Bad_Request, "Content error, empty")
 132         #    return
 133         if confidential_data:
 134             logger.info('IN: %s', remove_clear_passwd (yaml.safe_dump(client_data, explicit_start=True, indent=4, default_flow_style=False,
 135                                               tags=False, allow_unicode=True)))
 136         else:
 137             logger.info('IN: %s', yaml.safe_dump(client_data, explicit_start=True, indent=4, default_flow_style=False,
 138                                               tags=False, allow_unicode=True) )
 139         # look for the client provider version
 140         error_text = "Invalid content "
 141         if not default_schema and not version_fields:
 142             return client_data, None
 143         client_version = None
 144         used_schema = None
 145         if version_fields != None:
 146             client_version = client_data
 147             for field in version_fields:
 148                 if field in client_version:
 149                     client_version = client_version[field]
 150                 else:
 151                     client_version=None
 152                     break
 153         if client_version == None:
 154             used_schema = default_schema
 155         elif version_dict_schema != None:
 156             if client_version in version_dict_schema:
 157                 used_schema = version_dict_schema[client_version]
 158             elif None in version_dict_schema:
 159                 used_schema = version_dict_schema[None]
 160         if used_schema==None:
 161             bottle.abort(httperrors.Bad_Request, "Invalid schema version or missing version field")
 162
 163         js_v(client_data, used_schema)
 164         return client_data, used_schema
 165     except (TypeError, ValueError, yaml.YAMLError) as exc:
 166         error_text += str(exc)
 167         logger.error(error_text)
 168         bottle.abort(httperrors.Bad_Request, error_text)
 169     except js_e.ValidationError as exc:
 170         logger.error(
 171             "validate_in error, jsonschema exception")
 172         error_pos = ""
 173         if len(exc.path)>0: error_pos=" at " + ":".join(map(json.dumps, exc.path))
 174         bottle.abort(httperrors.Bad_Request, error_text + exc.message + error_pos)
 175     #except:
 176     #    bottle.abort(httperrors.Bad_Request, "Content error: Failed to parse Content-Type",  error_pos)
 177     #    raise
 178
 179 def filter_query_string(qs, http2db, allowed):
 180     '''Process query string (qs) checking that contains only valid tokens for avoiding SQL injection
 181     Attributes:
 182         'qs': bottle.FormsDict variable to be processed. None or empty is considered valid
 183         'http2db': dictionary with change from http API naming (dictionary key) to database naming(dictionary value)
 184         'allowed': list of allowed string tokens (API http naming). All the keys of 'qs' must be one of 'allowed'
 185     Return: A tuple with the (select,where,limit) to be use in a database query. All of then transformed to the database naming
 186         select: list of items to retrieve, filtered by query string 'field=token'. If no 'field' is present, allowed list is returned
 187         where: dictionary with key, value, taken from the query string token=value. Empty if nothing is provided
 188         limit: limit dictated by user with the query string 'limit'. 100 by default
 189     abort if not permited, using bottel.abort
 190     '''
 191     where={}
 192     limit=100
 193     select=[]
 194     #if type(qs) is not bottle.FormsDict:
 195     #    bottle.abort(httperrors.Internal_Server_Error, '!!!!!!!!!!!!!!invalid query string not a dictionary')
 196     #    #bottle.abort(httperrors.Internal_Server_Error, "call programmer")
 197     for k in qs:
 198         if k=='field':
 199             select += qs.getall(k)
 200             for v in select:
 201                 if v not in allowed:
 202                     bottle.abort(httperrors.Bad_Request, "Invalid query string at 'field="+v+"'")
 203         elif k=='limit':
 204             try:
 205                 limit=int(qs[k])
 206             except:
 207                 bottle.abort(httperrors.Bad_Request, "Invalid query string at 'limit="+qs[k]+"'")
 208         else:
 209             if k not in allowed:
 210                 bottle.abort(httperrors.Bad_Request, "Invalid query string at '"+k+"="+qs[k]+"'")
 211             if qs[k]!="null":  where[k]=qs[k]
 212             else: where[k]=None
 213     if len(select)==0: select += allowed
 214     #change from http api to database naming
 215     for i in range(0,len(select)):
 216         k=select[i]
 217         if http2db and k in http2db:
 218             select[i] = http2db[k]
 219     if http2db:
 220         change_keys_http2db(where, http2db)
 221     #print "filter_query_string", select,where,limit
 222
 223     return select,where,limit