durin42 / async-http
Async http library intended for use in httplib2. Probably buggy, will become part of httplib2 when it is ready.
Clone this repository (size: 61.7 KB): HTTPS / SSH
$ hg clone http://bitbucket.org/durin42/async-http/
| commit 46: | 0ba711e07bf9 |
| parent 45: | 50a458cf2774 |
| branch: | default |
http: fix a bug with incoming chunked-transfer encoding
We would fail to read the block properly if the read from the socket split the incoming buffer in the middle of the chunk length indicator
6 weeks ago
Changed (Δ4.4 KB):
raw changeset »
http.py (13 lines added, 1 lines removed)
test/simple_http_test.py (0 lines added, 26 lines removed)
test/test_chunked_transfer.py (75 lines added, 0 lines removed)
| … | … | @@ -64,6 +64,7 @@ class HTTPResponse(object): |
64 |
64 |
self._chunked_done = False |
65 |
65 |
self._chunked_until_next = 0 |
66 |
66 |
self._chunked_skip_bytes = 0 |
67 |
self._chunked_preloaded_block = None |
|
67 |
68 |
self.wait_for_continue = expect_continue |
68 |
69 |
|
69 |
70 |
self._read_location = 0 |
| … | … | @@ -119,6 +120,10 @@ class HTTPResponse(object): |
119 |
120 |
self._load_response(data) |
120 |
121 |
|
121 |
122 |
def _chunked_parsedata(self, data): |
123 |
if self._chunked_preloaded_block: |
|
124 |
oldlen = len(data) |
|
125 |
data = self._chunked_preloaded_block + data |
|
126 |
self._chunked_preloaded_block = None |
|
122 |
127 |
while data: |
123 |
128 |
logger.debug('looping with %d data remaining', len(data)) |
124 |
129 |
# Slice out anything we should skip |
| … | … | @@ -129,6 +134,7 @@ class HTTPResponse(object): |
129 |
134 |
break |
130 |
135 |
else: |
131 |
136 |
data = data[self._chunked_skip_bytes:] |
137 |
self._chunked_skip_bytes = 0 |
|
132 |
138 |
|
133 |
139 |
# determine how much is until the next chunk |
134 |
140 |
if self._chunked_until_next: |
| … | … | @@ -137,7 +143,13 @@ class HTTPResponse(object): |
137 |
143 |
self._chunked_until_next = 0 |
138 |
144 |
body = data |
139 |
145 |
else: |
140 |
|
|
146 |
try: |
|
147 |
amt, body = data.split(EOL, 1) |
|
148 |
except ValueError: |
|
149 |
self._chunked_preloaded_block = data |
|
150 |
logger.debug('saving %r as a preloaded block for chunked', |
|
151 |
self._chunked_preloaded_block) |
|
152 |
return |
|
141 |
153 |
amt = int(amt, base=16) |
142 |
154 |
logger.debug('reading chunk of length %d', amt) |
143 |
155 |
if amt == 0: |
Up to file-list test/simple_http_test.py:
| … | … | @@ -109,29 +109,3 @@ class SimpleHttpTest(util.HttpTestBase, |
109 |
109 |
self.assertEqual(expected_req, sock.sent) |
110 |
110 |
self.assertEqual("You can do that.", con.getresponse().read()) |
111 |
111 |
self.assertEqual(sock.closed, False) |
112 |
||
113 |
def testChunkedTransfer(self): |
|
114 |
con = http.HTTPConnection('1.2.3.4:80') |
|
115 |
con._connect() |
|
116 |
sock = con.sock |
|
117 |
sock.read_wait_sentinel = 'end-of-body' |
|
118 |
sock.data = ['HTTP/1.1 200 OK\r\n', |
|
119 |
'Server: BogusServer 1.0\r\n', |
|
120 |
'Content-Length: 6', |
|
121 |
'\r\n\r\n', |
|
122 |
"Thanks"] |
|
123 |
||
124 |
zz = 'zz\n' |
|
125 |
con.request('POST', '/', body=cStringIO.StringIO((zz * (0x8010/3)) + 'end-of-body')) |
|
126 |
expected_req = ('POST / HTTP/1.1\r\n' |
|
127 |
'transfer-encoding: chunked\r\n' |
|
128 |
'Host: 1.2.3.4\r\n' |
|
129 |
'accept-encoding: identity\r\n\r\n') |
|
130 |
expected_req += '8000\r\n%szz\r\n' % ('zz\n' * (0x8000 / 3)) |
|
131 |
expected_req += '1b\r\n\n%s\r\n' % ('zz\n' * ((0x1b - len('end-of-body')) / 3) |
|
132 |
+ 'end-of-body') |
|
133 |
expected_req += '0\r\n\r\n' |
|
134 |
self.assertEqual(('1.2.3.4', 80), sock.sa) |
|
135 |
self.assertStringEqual(expected_req, sock.sent) |
|
136 |
self.assertEqual("Thanks", con.getresponse().read()) |
|
137 |
self.assertEqual(sock.closed, False) |
Up to file-list test/test_chunked_transfer.py:
1 |
import cStringIO |
|
2 |
import doctest |
|
3 |
import unittest |
|
4 |
||
5 |
import http |
|
6 |
import http_socket |
|
7 |
from test import util |
|
8 |
||
9 |
||
10 |
def chunkedblock(x): |
|
11 |
"""Make a chunked transfer-encoding block. |
|
12 |
||
13 |
>>> chunkedblock('hi') |
|
14 |
'2\r\nhi\r\n' |
|
15 |
>>> chunkedblock('hi' * 10) |
|
16 |
'14\r\nhihihihihihihihihihi\r\n' |
|
17 |
""" |
|
18 |
return '%s\r\n%s\r\n' % (hex(len(x))[2:], x) |
|
19 |
||
20 |
||
21 |
class ChunkedTransferTest(util.HttpTestBase, unittest.TestCase): |
|
22 |
def testChunkedUpload(self): |
|
23 |
con = http.HTTPConnection('1.2.3.4:80') |
|
24 |
con._connect() |
|
25 |
sock = con.sock |
|
26 |
sock.read_wait_sentinel = 'end-of-body' |
|
27 |
sock.data = ['HTTP/1.1 200 OK\r\n', |
|
28 |
'Server: BogusServer 1.0\r\n', |
|
29 |
'Content-Length: 6', |
|
30 |
'\r\n\r\n', |
|
31 |
"Thanks"] |
|
32 |
||
33 |
zz = 'zz\n' |
|
34 |
con.request('POST', '/', body=cStringIO.StringIO((zz * (0x8010/3)) + 'end-of-body')) |
|
35 |
expected_req = ('POST / HTTP/1.1\r\n' |
|
36 |
'transfer-encoding: chunked\r\n' |
|
37 |
'Host: 1.2.3.4\r\n' |
|
38 |
'accept-encoding: identity\r\n\r\n') |
|
39 |
expected_req += chunkedblock('zz\n' * (0x8000 / 3) + 'zz') |
|
40 |
expected_req += chunkedblock('\n' + 'zz\n' * ((0x1b - len('end-of-body')) / 3) |
|
41 |
+ 'end-of-body') |
|
42 |
expected_req += '0\r\n\r\n' |
|
43 |
self.assertEqual(('1.2.3.4', 80), sock.sa) |
|
44 |
self.assertStringEqual(expected_req, sock.sent) |
|
45 |
self.assertEqual("Thanks", con.getresponse().read()) |
|
46 |
self.assertEqual(sock.closed, False) |
|
47 |
||
48 |
def testChunkedDownload(self): |
|
49 |
con = http.HTTPConnection('1.2.3.4:80') |
|
50 |
con._connect() |
|
51 |
sock = con.sock |
|
52 |
sock.data = ['HTTP/1.1 200 OK\r\n', |
|
53 |
'Server: BogusServer 1.0\r\n', |
|
54 |
'transfer-encoding: chunked', |
|
55 |
'\r\n\r\n', |
|
56 |
chunkedblock('hi '), |
|
57 |
chunkedblock('there'), |
|
58 |
chunkedblock(''), |
|
59 |
] |
|
60 |
con.request('GET', '/') |
|
61 |
self.assertStringEqual('hi there', con.getresponse().read()) |
|
62 |
||
63 |
def testChunkedDownloadPartialChunk(self): |
|
64 |
con = http.HTTPConnection('1.2.3.4:80') |
|
65 |
con._connect() |
|
66 |
sock = con.sock |
|
67 |
sock.data = ['HTTP/1.1 200 OK\r\n', |
|
68 |
'Server: BogusServer 1.0\r\n', |
|
69 |
'transfer-encoding: chunked', |
|
70 |
'\r\n\r\n', |
|
71 |
chunkedblock('hi '), |
|
72 |
] + list(chunkedblock('there\n' * 5)) + [chunkedblock('')] |
|
73 |
con.request('GET', '/') |
|
74 |
self.assertStringEqual('hi there\nthere\nthere\nthere\nthere\n', |
|
75 |
con.getresponse().read()) |