3 import urllib
.parse
as urlparse
5 import http
.cookies
as Cookie
10 import tornado
.httpserver
12 import tornado
.iostream
14 import tornado
.httpclient
17 __all__
= ['run_proxy', 'RequestObj', 'ResponseObj']
19 DEFAULT_CALLBACK
= lambda r
: r
23 def __init__(self
, **kwds
):
24 self
.__dict
__.update(kwds
)
27 return str(self
.__dict
__)
30 class RequestObj(Bunch
):
32 An HTTP request object that contains the following request attributes:
34 protocol: either 'http' or 'https'
35 host: the destination hostname of the request
36 port: the port for the request
37 path: the path of the request ('/index.html' for example)
38 query: the query string ('?key=value&other=value')
39 fragment: the hash fragment ('#fragment')
40 method: request method ('GET', 'POST', etc)
41 username: always passed as None, but you can set it to override the user
42 password: None, but can be set to override the password
43 body: request body as a string
44 headers: a dictionary of header / value pairs
45 (for example {'Content-Type': 'text/plain', 'Content-Length': 200})
46 follow_redirects: true to follow redirects before returning a response
47 validate_cert: false to turn off SSL cert validation
48 context: a dictionary to place data that will be accessible to the response
53 class ResponseObj(Bunch
):
55 An HTTP response object that contains the following request attributes:
57 code: response code, such as 200 for 'OK'
58 headers: the response headers
59 pass_headers: a list or set of headers to pass along in the response. All
60 other headeres will be stripped out. By default this includes:
61 ('Date', 'Cache-Control', 'Server', 'Content-Type', 'Location')
62 body: response body as a string
63 context: the context object from the request
66 def __init__(self
, **kwargs
):
67 kwargs
.setdefault('code', 200)
68 kwargs
.setdefault('headers', {})
69 kwargs
.setdefault('pass_headers', True)
70 kwargs
.setdefault('body', '')
71 kwargs
.setdefault('context', {})
72 super(ResponseObj
, self
).__init
__(**kwargs
)
75 class ResponseStreamWriterFuture(tornado
.concurrent
.Future
):
76 def __init__(self
, write_fn
, *args
, **kwargs
):
77 self
.write_fn
= write_fn
78 super().__init
__(*args
, **kwargs
)
81 def _make_proxy(methods
, io_loop
, req_callback
, resp_callback
, err_callback
, debug_level
=0):
83 @tornado.web
.stream_request_body
84 class ProxyHandler(tornado
.web
.RequestHandler
):
86 SUPPORTED_METHODS
= methods
89 self
.proxy_request_ready
= tornado
.concurrent
.Future()
90 self
.request_future
= None
92 def on_connection_close(self
):
93 if self
.request_future
is not None:
94 self
.request_future
.set_result(False)
96 def create_body_producer_future(self
, write_fn
):
97 self
.request_future
= ResponseStreamWriterFuture(write_fn
)
98 self
.proxy_request_ready
.set_result(True)
99 return self
.request_future
101 @tornado.gen
.coroutine
102 def data_received(self
, chunk
):
103 yield self
.proxy_request_ready
105 yield self
.request_future
.write_fn(chunk
)
107 def make_requestobj(self
, request
):
109 creates a request object for this request
112 # get url for request
113 # surprisingly, tornado's HTTPRequest sometimes
114 # has a uri field with the full uri (http://...)
115 # and sometimes it just contains the path. :(
118 if not url
.startswith(u
'http'):
119 url
= u
"{proto}://{netloc}{path}".format(
120 proto
=request
.protocol
,
125 parsedurl
= urlparse
.urlparse(url
)
127 # Passing on the transfer encoding header, causes Tornado to not
128 # transmit valid chunks
129 headers
= request
.headers
.copy()
130 if "Transfer-encoding" in headers
:
131 del headers
["Transfer-Encoding"]
133 # create request object
135 requestobj
= RequestObj(
136 method
=request
.method
,
137 protocol
=parsedurl
.scheme
,
140 host
=parsedurl
.hostname
,
141 port
=parsedurl
.port
or 80,
143 query
=parsedurl
.query
,
144 fragment
=parsedurl
.fragment
,
147 follow_redirects
=False,
152 return requestobj
, parsedurl
155 def make_request(self
, obj
, parsedurl
):
157 converts a request object into an HTTPRequest
160 obj
.headers
.setdefault('Host', obj
.host
)
162 if obj
.username
or parsedurl
.username
or \
163 obj
.password
or parsedurl
.password
:
165 auth
= u
"{username}:{password}@".format(
166 username
=obj
.username
or parsedurl
.username
,
167 password
=obj
.password
or parsedurl
.password
173 url
= u
"{proto}://{auth}{host}{port}{path}{query}{frag}"
178 port
=(u
':' + str(obj
.port
)) if (obj
.port
and obj
.port
!= 80) else u
'',
179 path
=u
'/'+obj
.path
.lstrip(u
'/') if obj
.path
else u
'',
180 query
=u
'?'+obj
.query
.lstrip(u
'?') if obj
.query
else u
'',
185 if "Transfer-encoding" in self
.request
.headers
and \
186 self
.request
.headers
["Transfer-Encoding"] == "chunked":
187 body_producer
= self
.create_body_producer_future
189 req
= tornado
.httpclient
.HTTPRequest(
192 body_producer
=body_producer
,
193 decompress_response
=False,
195 follow_redirects
=obj
.follow_redirects
,
196 allow_nonstandard_methods
=True,
197 request_timeout
=1*60*60 #1 hour
204 request
= self
.request
206 print("<<<<<<<< REQUEST <<<<<<<<")
207 pprint
.pprint(request
.__dict
__)
212 MAX_STREAMED_SIZE
= 50 * GB
213 request
.connection
.set_max_body_size(MAX_STREAMED_SIZE
)
215 requestobj
, parsedurl
= self
.make_requestobj(request
)
218 print("<<<<<<<< REQUESTOBJ <<<<<<<<")
219 pprint
.pprint(requestobj
.__dict
__)
222 debugstr
= "serving request from %s:%d%s " % (requestobj
.host
,
223 requestobj
.port
or 80,
226 modrequestobj
= req_callback(requestobj
)
228 if isinstance(modrequestobj
, ResponseObj
):
229 self
.handle_response(modrequestobj
)
233 print(debugstr
+ "to %s:%d%s" % (modrequestobj
.host
,
234 modrequestobj
.port
or 80,
237 outreq
= self
.make_request(modrequestobj
, parsedurl
)
240 print(">>>>>>>> REQUEST >>>>>>>>")
241 print("%s %s" % (outreq
.method
, outreq
.url
))
242 for k
, v
in outreq
.headers
.items():
243 print( "%s: %s" % (k
, v
))
247 def _resp_callback(response
):
248 self
.handle_response(response
, context
=modrequestobj
.context
)
250 client
= tornado
.httpclient
.AsyncHTTPClient(io_loop
=io_loop
)
252 client
.fetch(outreq
, _resp_callback
,
253 validate_cert
=modrequestobj
.validate_cert
)
254 except tornado
.httpclient
.HTTPError
as e
:
255 if hasattr(e
, 'response') and e
.response
:
256 self
.handle_response(e
.response
,
257 context
=modrequestobj
.context
,
261 self
.write('Internal server error:\n' + str(e
))
265 def handle_response(self
, response
, context
={}, error
=False):
267 if not isinstance(response
, ResponseObj
):
269 print("<<<<<<<< RESPONSE <<<<<<<")
270 pprint
.pprint(response
.__dict
__)
272 responseobj
= ResponseObj(
274 headers
=response
.headers
,
280 responseobj
= response
283 print("<<<<<<<< RESPONSEOBJ <<<<<<<")
284 responseprint
= copy(responseobj
)
285 responseprint
.body
= "-- body content not displayed --"
286 pprint
.pprint(responseprint
.__dict
__)
289 mod
= resp_callback(responseobj
)
291 mod
= err_callback(responseobj
)
293 # set the response status code
297 self
.write('Internal server error. Server unreachable.')
301 self
.set_status(mod
.code
)
303 # set the response headers
305 if type(mod
.pass_headers
) == bool:
306 header_keys
= mod
.headers
.keys() if mod
.pass_headers
else []
308 header_keys
= mod
.pass_headers
309 for key
in header_keys
:
310 if key
.lower() == "set-cookie":
311 cookies
= Cookie
.BaseCookie()
312 cookies
.load(tornado
.escape
.native_str(mod
.headers
.get(key
)))
313 for cookie_key
in cookies
:
314 cookie
= cookies
[cookie_key
]
315 params
= dict(cookie
)
316 expires
= params
.pop('expires', None)
318 expires
= dateutil
.parser
.parse(expires
)
326 val
= mod
.headers
.get(key
)
327 self
.set_header(key
, val
)
330 print(">>>>>>>> RESPONSE (%s) >>>>>>>" % mod
.code
)
331 for k
, v
in self
._headers
.items():
332 print("%s: %s" % (k
, v
))
333 if hasattr(self
, '_new_cookie'):
334 print(self
._new
_cookie
.output())
336 # set the response body
343 @tornado.web
.asynchronous
347 @tornado.web
.asynchronous
351 @tornado.web
.asynchronous
355 @tornado.web
.asynchronous
357 self
.request_future
.set_result(True)
359 @tornado.web
.asynchronous
361 self
.request_future
.set_result(True)
363 @tornado.web
.asynchronous
365 self
.request_future
.set_result(True)
367 @tornado.web
.asynchronous
376 methods
=['GET', 'POST', 'PUT', 'DELETE', 'HEAD'],
377 req_callback
=DEFAULT_CALLBACK
,
378 resp_callback
=DEFAULT_CALLBACK
,
379 err_callback
=DEFAULT_CALLBACK
,
387 Run proxy on the specified port.
389 methods: the HTTP methods this proxy will support
390 req_callback: a callback that is passed a RequestObj that it should
391 modify and then return
392 resp_callback: a callback that is given a ResponseObj that it should
393 modify and then return
394 err_callback: in the case of an error, this callback will be called.
395 there's no difference between how this and the resp_callback are
397 test_ssl: if true, will wrap the socket in an self signed ssl cert
398 start_ioloop: if True (default), the tornado IOLoop will be started
400 debug_level: 0 no debug, 1 basic, 2 verbose
403 io_loop
= tornado
.ioloop
.IOLoop
.instance() if io_loop
is None else io_loop
405 app
= tornado
.web
.Application([
406 (r
'.*', _make_proxy(methods
=methods
,
408 req_callback
=req_callback
,
409 resp_callback
=resp_callback
,
410 err_callback
=err_callback
,
411 debug_level
=debug_level
)),
415 this_dir
, this_filename
= os
.path
.split(__file__
)
418 "certfile": os
.path
.join(this_dir
, "data", "test.crt"),
419 "keyfile": os
.path
.join(this_dir
, "data", "test.key"),
424 kwargs
= {"io_loop": io_loop
}
426 http_server
= tornado
.httpserver
.HTTPServer(app
, **kwargs
)
427 http_server
.listen(port
, address
)
431 if __name__
== '__main__':
433 if len(sys
.argv
) > 1:
434 port
= int(sys
.argv
[1])
436 print("Starting HTTP proxy on port %d" % port
)