Merge from master (July 26th, 2019)

CHANGES.rst

@@ -37,6 +37,9 @@ dev (master)
 * Fix issue where URLs containing invalid characters within ``Url.auth`` would
   raise an exception instead of percent-encoding those characters.
 
+* Add support for ``HTTPResponse.auto_close = False`` which makes HTTP responses
+  work well with BufferedReaders and other ``io`` module features. (Pull #1652)
+
 
 1.25.3 (2019-05-23)
 -------------------

docs/user-guide.rst

@@ -77,6 +77,20 @@ to a byte string representing the response content::
 .. note:: For larger responses, it's sometimes better to :ref:`stream <stream>`
     the response.
 
+Using io Wrappers with Response content
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Sometimes you want to use :class:`io.TextIOWrapper` or similar objects like a CSV reader
+directly with :class:`~response.HTTPResponse` data. Making these two interfaces play nice
+together requires using the :attr:`~response.HTTPResponse.auto_close` attribute by setting it
+to ``False``. By default HTTP responses are closed after reading all bytes, this disables that behavior::
+
+    >>> import io
+    >>> r = http.request('GET', 'https://example.com', preload_content=False)
+    >>> r.auto_close = False
+    >>> for line in io.TextIOWrapper(r):
+    >>>     print(line)
+
 .. _request_data:
 
 Request data

test/test_response.py

@@ -1,8 +1,12 @@
+# -*- coding: utf-8 -*-
+
+import re
 import zlib
 
-from io import BytesIO, BufferedReader
+from io import BytesIO, BufferedReader, TextIOWrapper
 
 import pytest
+import six
 
 from urllib3.base import Response
 from urllib3.response import HTTPResponse, brotli
@@ -305,6 +309,14 @@ def test_io_bufferedreader(self):
         br.close()
         assert resp.closed
 
+        # HTTPResponse.read() by default closes the response
+        # https://github.com/urllib3/urllib3/issues/1305
+        fp = BytesIO(b"hello\nworld")
+        resp = HTTPResponse(fp, preload_content=False)
+        with pytest.raises(ValueError) as ctx:
+            list(BufferedReader(resp))
+        assert str(ctx.value) == "readline of closed file"
+
         b = b"!tenbytes!"
         fp = BytesIO(b)
         resp = HTTPResponse(fp, preload_content=False)
@@ -316,6 +328,30 @@ def test_io_bufferedreader(self):
         assert len(br.read(5)) == 5
         assert len(br.read(5)) == 0
 
+    def test_io_textiowrapper(self):
+        fp = BytesIO(b"\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f")
+        resp = HTTPResponse(fp, preload_content=False)
+        br = TextIOWrapper(resp, encoding="utf8")
+
+        assert br.read() == u"äöüß"
+
+        br.close()
+        assert resp.closed
+
+        # HTTPResponse.read() by default closes the response
+        # https://github.com/urllib3/urllib3/issues/1305
+        fp = BytesIO(
+            b"\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f\n\xce\xb1\xce\xb2\xce\xb3\xce\xb4"
+        )
+        resp = HTTPResponse(fp, preload_content=False)
+        with pytest.raises(ValueError) as ctx:
+            if six.PY2:
+                # py2's implementation of TextIOWrapper requires `read1`
+                # method which is provided by `BufferedReader` wrapper
+                resp = BufferedReader(resp)
+            list(TextIOWrapper(resp))
+        assert re.match("I/O operation on closed file.?", str(ctx.value))
+
     def test_streaming(self):
         fp = [b"fo", b"o"]
         resp = HTTPResponse(fp, preload_content=False)

test/test_ssl.py

@@ -1,7 +1,8 @@
-import mock
 import platform
 import sys
 
+import mock
+
 import pytest
 from urllib3.util import ssl_
 from urllib3.exceptions import SNIMissingWarning
@@ -131,7 +132,6 @@ def test_wrap_socket_given_ca_certs_no_load_default_certs(monkeypatch):
     if platform.python_implementation() == "PyPy" and sys.version_info[0] == 2:
         # https://github.com/testing-cabal/mock/issues/438
         pytest.xfail("fails with PyPy for Python 2 dues to funcsigs bug")
-
     context = mock.create_autospec(ssl_.SSLContext)
     context.load_default_certs = mock.Mock()
     context.options = 0