[Python-checkins] r87399 - in python/branches/py3k: Doc/library/http.client.rst Doc/library/urllib.request.rst Lib/http/client.py Lib/test/test_httplib.py Lib/test/test_urllib2.py Lib/urllib/request.py Misc/NEWS
senthil.kumaran
python-checkins at python.org
Sun Dec 19 11:49:52 CET 2010
Author: senthil.kumaran
Date: Sun Dec 19 11:49:52 2010
New Revision: 87399
Log:
Issue3243 - Support iterable bodies in httplib. Patch contributions by Xuanji Li and Chris AtLee.
Modified:
python/branches/py3k/Doc/library/http.client.rst
python/branches/py3k/Doc/library/urllib.request.rst
python/branches/py3k/Lib/http/client.py
python/branches/py3k/Lib/test/test_httplib.py
python/branches/py3k/Lib/test/test_urllib2.py
python/branches/py3k/Lib/urllib/request.py
python/branches/py3k/Misc/NEWS
Modified: python/branches/py3k/Doc/library/http.client.rst
==============================================================================
--- python/branches/py3k/Doc/library/http.client.rst (original)
+++ python/branches/py3k/Doc/library/http.client.rst Sun Dec 19 11:49:52 2010
@@ -393,14 +393,18 @@
string.
The *body* may also be an open :term:`file object`, in which case the
- contents of the file is sent; this file object should support
- ``fileno()`` and ``read()`` methods. The header Content-Length is
- automatically set to the length of the file as reported by
- stat.
+ contents of the file is sent; this file object should support ``fileno()``
+ and ``read()`` methods. The header Content-Length is automatically set to
+ the length of the file as reported by stat. The *body* argument may also be
+ an iterable and Contet-Length header should be explicitly provided when the
+ body is an iterable.
The *headers* argument should be a mapping of extra HTTP
headers to send with the request.
+ .. versionadded:: 3.2
+ *body* can be an iterable
+
.. method:: HTTPConnection.getresponse()
Should be called after a request is sent to get the response from the server.
Modified: python/branches/py3k/Doc/library/urllib.request.rst
==============================================================================
--- python/branches/py3k/Doc/library/urllib.request.rst (original)
+++ python/branches/py3k/Doc/library/urllib.request.rst Sun Dec 19 11:49:52 2010
@@ -21,13 +21,14 @@
:class:`Request` object.
*data* may be a string specifying additional data to send to the
- server, or ``None`` if no such data is needed. Currently HTTP
- requests are the only ones that use *data*; the HTTP request will
- be a POST instead of a GET when the *data* parameter is provided.
- *data* should be a buffer in the standard
+ server, or ``None`` if no such data is needed. *data* may also be an
+ iterable object and in that case Content-Length value must be specified in
+ the headers. Currently HTTP requests are the only ones that use *data*; the
+ HTTP request will be a POST instead of a GET when the *data* parameter is
+ provided. *data* should be a buffer in the standard
:mimetype:`application/x-www-form-urlencoded` format. The
- :func:`urllib.parse.urlencode` function takes a mapping or sequence
- of 2-tuples and returns a string in this format. urllib.request module uses
+ :func:`urllib.parse.urlencode` function takes a mapping or sequence of
+ 2-tuples and returns a string in this format. urllib.request module uses
HTTP/1.1 and includes ``Connection:close`` header in its HTTP requests.
The optional *timeout* parameter specifies a timeout in seconds for
@@ -76,6 +77,9 @@
HTTPS virtual hosts are now supported if possible (that is, if
:data:`ssl.HAS_SNI` is true).
+ .. versionadded:: 3.2
+ *data* can be an iterable object.
+
.. function:: install_opener(opener)
Install an :class:`OpenerDirector` instance as the default global opener.
Modified: python/branches/py3k/Lib/http/client.py
==============================================================================
--- python/branches/py3k/Lib/http/client.py (original)
+++ python/branches/py3k/Lib/http/client.py Sun Dec 19 11:49:52 2010
@@ -71,6 +71,7 @@
import io
import os
import socket
+import collections
from urllib.parse import urlsplit
import warnings
@@ -730,7 +731,11 @@
self.__state = _CS_IDLE
def send(self, data):
- """Send `data' to the server."""
+ """Send `data' to the server.
+ ``data`` can be a string object, a bytes object, an array object, a
+ file-like object that supports a .read() method, or an iterable object.
+ """
+
if self.sock is None:
if self.auto_open:
self.connect()
@@ -762,8 +767,16 @@
if encode:
datablock = datablock.encode("iso-8859-1")
self.sock.sendall(datablock)
- else:
+
+ try:
self.sock.sendall(data)
+ except TypeError:
+ if isinstance(data, collections.Iterable):
+ for d in data:
+ self.sock.sendall(d)
+ else:
+ raise TypeError("data should be byte-like object\
+ or an iterable, got %r " % type(it))
def _output(self, s):
"""Add a line of output to the current request buffer.
Modified: python/branches/py3k/Lib/test/test_httplib.py
==============================================================================
--- python/branches/py3k/Lib/test/test_httplib.py (original)
+++ python/branches/py3k/Lib/test/test_httplib.py Sun Dec 19 11:49:52 2010
@@ -230,6 +230,22 @@
conn.send(io.BytesIO(expected))
self.assertEqual(expected, sock.data)
+ def test_send_iter(self):
+ expected = b'GET /foo HTTP/1.1\r\nHost: example.com\r\n' \
+ b'Accept-Encoding: identity\r\nContent-Length: 11\r\n' \
+ b'\r\nonetwothree'
+
+ def body():
+ yield b"one"
+ yield b"two"
+ yield b"three"
+
+ conn = client.HTTPConnection('example.com')
+ sock = FakeSocket("")
+ conn.sock = sock
+ conn.request('GET', '/foo', body(), {'Content-Length': '11'})
+ self.assertEquals(sock.data, expected)
+
def test_chunked(self):
chunked_start = (
'HTTP/1.1 200 OK\r\n'
Modified: python/branches/py3k/Lib/test/test_urllib2.py
==============================================================================
--- python/branches/py3k/Lib/test/test_urllib2.py (original)
+++ python/branches/py3k/Lib/test/test_urllib2.py Sun Dec 19 11:49:52 2010
@@ -4,6 +4,7 @@
import os
import io
import socket
+import array
import urllib.request
from urllib.request import Request, OpenerDirector
@@ -765,7 +766,7 @@
o = h.parent = MockOpener()
url = "http://example.com/"
- for method, data in [("GET", None), ("POST", "blah")]:
+ for method, data in [("GET", None), ("POST", b"blah")]:
req = Request(url, data, {"Foo": "bar"})
req.timeout = None
req.add_unredirected_header("Spam", "eggs")
@@ -795,7 +796,7 @@
# check adding of standard headers
o.addheaders = [("Spam", "eggs")]
- for data in "", None: # POST, GET
+ for data in b"", None: # POST, GET
req = Request("http://example.com/", data)
r = MockResponse(200, "OK", {}, "")
newreq = h.do_request_(req)
@@ -821,6 +822,50 @@
self.assertEqual(req.unredirected_hdrs["Host"], "baz")
self.assertEqual(req.unredirected_hdrs["Spam"], "foo")
+ # Check iterable body support
+ def iterable_body():
+ yield b"one"
+ yield b"two"
+ yield b"three"
+
+ for headers in {}, {"Content-Length": 11}:
+ req = Request("http://example.com/", iterable_body(), headers)
+ if not headers:
+ # Having an iterable body without a Content-Length should
+ # raise an exception
+ self.assertRaises(ValueError, h.do_request_, req)
+ else:
+ newreq = h.do_request_(req)
+
+ # A file object
+
+ """
+ file_obj = io.StringIO()
+ file_obj.write("Something\nSomething\nSomething\n")
+
+ for headers in {}, {"Content-Length": 30}:
+ req = Request("http://example.com/", file_obj, headers)
+ if not headers:
+ # Having an iterable body without a Content-Length should
+ # raise an exception
+ self.assertRaises(ValueError, h.do_request_, req)
+ else:
+ newreq = h.do_request_(req)
+ self.assertEqual(int(newreq.get_header('Content-length')),30)
+
+ file_obj.close()
+
+ # array.array Iterable - Content Length is calculated
+
+ iterable_array = array.array("I",[1,2,3,4])
+
+ for headers in {}, {"Content-Length": 16}:
+ req = Request("http://example.com/", iterable_array, headers)
+ newreq = h.do_request_(req)
+ self.assertEqual(int(newreq.get_header('Content-length')),16)
+ """
+
+
def test_http_doubleslash(self):
# Checks the presence of any unnecessary double slash in url does not
# break anything. Previously, a double slash directly after the host
@@ -828,7 +873,7 @@
h = urllib.request.AbstractHTTPHandler()
o = h.parent = MockOpener()
- data = ""
+ data = b""
ds_urls = [
"http://example.com/foo/bar/baz.html",
"http://example.com//foo/bar/baz.html",
Modified: python/branches/py3k/Lib/urllib/request.py
==============================================================================
--- python/branches/py3k/Lib/urllib/request.py (original)
+++ python/branches/py3k/Lib/urllib/request.py Sun Dec 19 11:49:52 2010
@@ -94,6 +94,7 @@
import socket
import sys
import time
+import collections
from urllib.error import URLError, HTTPError, ContentTooShortError
from urllib.parse import (
@@ -1053,8 +1054,17 @@
'Content-type',
'application/x-www-form-urlencoded')
if not request.has_header('Content-length'):
- request.add_unredirected_header(
- 'Content-length', '%d' % len(data))
+ try:
+ mv = memoryview(data)
+ except TypeError:
+ print(data)
+ if isinstance(data, collections.Iterable):
+ raise ValueError("Content-Length should be specified \
+ for iterable data of type %r %r" % (type(data),
+ data))
+ else:
+ request.add_unredirected_header(
+ 'Content-length', '%d' % len(mv) * mv.itemsize)
sel_host = host
if request.has_proxy():
Modified: python/branches/py3k/Misc/NEWS
==============================================================================
--- python/branches/py3k/Misc/NEWS (original)
+++ python/branches/py3k/Misc/NEWS Sun Dec 19 11:49:52 2010
@@ -23,6 +23,9 @@
Library
-------
+- Issue #3243: Support iterable bodies in httplib. Patch Contributions by
+ Xuanji Li and Chris AtLee.
+
- Issue #10611: SystemExit exception will no longer kill a unittest run.
- Issue #9857: It is now possible to skip a test in a setUp, tearDown or clean
More information about the Python-checkins
mailing list