Skip to content

Commit afc3079

Browse files
committed
gh-49288: Resend POST body on HTTP auth retry
Fix :mod:`urllib.request` HTTP authentication retries sending an empty POST body when :attr:`~urllib.request.Request.data` is a file-like object. The body is now rewound to the position it was in before the original request was sent; if it is not seekable, the retry is abandoned and the ``401`` response is returned to the caller instead.
1 parent eb21dea commit afc3079

4 files changed

Lines changed: 353 additions & 4 deletions

File tree

‎Doc/library/urllib.request.rst‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1099,6 +1099,9 @@ HTTPBasicAuthHandler Objects
10991099

11001100
Retry the request with authentication information, if available.
11011101

1102+
The :attr:`Request.data` of a POST or PUT request is resent. However,
1103+
if it is a file-like object that is not seekable, the retry is
1104+
abandoned and the ``401`` response is returned to the caller instead.
11021105

11031106
.. _proxy-basic-auth-handler:
11041107

@@ -1146,6 +1149,10 @@ HTTPDigestAuthHandler Objects
11461149

11471150
Retry the request with authentication information, if available.
11481151

1152+
The :attr:`Request.data` of a POST or PUT request is resent. However,
1153+
if it is a file-like object that is not seekable, the retry is
1154+
abandoned and the ``401`` response is returned to the caller instead.
1155+
11491156

11501157
.. _proxy-digest-auth-handler:
11511158

‎Lib/test/test_urllib2.py‎

Lines changed: 309 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import os
1010
import io
1111
import hashlib
12+
import mmap
1213
import ftplib
1314
import socket
1415
import array
@@ -19,10 +20,11 @@
1920
import urllib.request
2021
# The proxy bypass method imported below has logic specific to the
2122
# corresponding system but is testable on all platforms.
22-
from urllib.request import (Request, OpenerDirector, HTTPBasicAuthHandler,
23-
HTTPPasswordMgrWithPriorAuth, _parse_proxy,
24-
_proxy_bypass_winreg_override,
25-
_proxy_bypass_macosx_sysconf,
23+
from urllib.request import (HTTPDefaultErrorHandler, HTTPErrorProcessor,
24+
Request, OpenerDirector, HTTPBasicAuthHandler,
25+
HTTPPasswordMgrWithPriorAuth,
26+
_parse_proxy, _proxy_bypass_winreg_override,
27+
_proxy_bypass_macosx_sysconf, _rewind_auth_retry_body,
2628
AbstractDigestAuthHandler)
2729
from urllib.parse import urlsplit
2830
import urllib.error
@@ -672,6 +674,113 @@ def http_open(self, req):
672674
return MockResponse(self.code, name, MockFile(), "", req.get_full_url())
673675

674676

677+
class _AuthRetrySocket:
678+
def __init__(self, response, sent_requests):
679+
self._response = io.BytesIO(response)
680+
self._sent_requests = sent_requests
681+
self._closed = False
682+
683+
def sendall(self, data):
684+
self._sent_requests.append(data)
685+
686+
def makefile(self, mode='r', buffering=None, *, encoding=None, errors=None):
687+
return self
688+
689+
def read(self, amt=-1):
690+
if self._closed:
691+
return b""
692+
return self._response.read(amt)
693+
694+
def readline(self, length=-1):
695+
if self._closed:
696+
return b""
697+
return self._response.readline(length)
698+
699+
def flush(self):
700+
self._response.flush()
701+
702+
def close(self):
703+
self._closed = True
704+
705+
def _open_with_auth_retry(auth_handler, http_handler_cls, conn_cls, url, data):
706+
conn_cls.sent_requests.clear()
707+
conn_cls._request_count = 0
708+
opener = OpenerDirector()
709+
opener.add_handler(auth_handler)
710+
opener.add_handler(http_handler_cls())
711+
opener.add_handler(HTTPErrorProcessor())
712+
opener.add_handler(HTTPDefaultErrorHandler())
713+
opener.open(Request(url, data, method="POST"))
714+
return conn_cls.sent_requests
715+
716+
def _decode_chunked_http_body(raw):
717+
body = raw
718+
result = b""
719+
while body:
720+
size_line, sep, body = body.partition(b"\r\n")
721+
if not sep:
722+
break
723+
size = int(size_line.split(b";", 1)[0], 16)
724+
if size == 0:
725+
break
726+
result += body[:size]
727+
body = body[size+2:]
728+
return result
729+
730+
def _group_sent_by_request(sent):
731+
groups = []
732+
current = []
733+
for chunk in sent:
734+
if chunk.startswith(b"POST ") and current:
735+
groups.append(current)
736+
current = [chunk]
737+
else:
738+
current.append(chunk)
739+
740+
if current:
741+
groups.append(current)
742+
return groups
743+
744+
def _body_from_request_chunks(chunks):
745+
return _body_from_sent_http_request(b"".join(chunks))
746+
747+
def _body_from_sent_http_request(raw):
748+
_, sep, body = raw.partition(b"\r\n\r\n")
749+
if not sep:
750+
return b""
751+
header_block = raw[:raw.index(sep)].lower()
752+
if b"transfer-encoding: chunked" in header_block:
753+
return _decode_chunked_http_body(body)
754+
if b"content-length: 0" in header_block:
755+
return b""
756+
return body.split(b"\r\n", 1)[0]
757+
758+
def _make_auth_retry_http_connection(responses):
759+
class AuthRetryHTTPConnection(http.client.HTTPConnection):
760+
sent_requests = []
761+
_request_count = 0
762+
763+
def connect(self):
764+
request_index = type(self)._request_count
765+
type(self)._request_count += 1
766+
response = responses[request_index]
767+
self.sock = _AuthRetrySocket(response, type(self).sent_requests)
768+
769+
return AuthRetryHTTPConnection
770+
771+
def _make_auth_retry_http_handler(responses):
772+
conn_cls = _make_auth_retry_http_connection(responses)
773+
774+
class AuthRetryHTTPHandler(urllib.request.HTTPHandler):
775+
def __init__(self):
776+
super().__init__()
777+
self.httpconn = conn_cls
778+
779+
def http_open(self, req):
780+
return self.do_open(self.httpconn, req)
781+
782+
return AuthRetryHTTPHandler, conn_cls
783+
675784

676785
class MockPasswordManager:
677786
def add_password(self, realm, uri, user, password):
@@ -2388,6 +2497,202 @@ def test_basic_prior_auth_send_after_first_success(self):
23882497
# expect request to be sent with auth header
23892498
self.assertTrue(http_handler.has_auth_header)
23902499

2500+
def test_rewind_auth_retry_body_no_data(self):
2501+
req = Request("http://example.com/")
2502+
self.assertTrue(_rewind_auth_retry_body(req))
2503+
2504+
def test_rewind_auth_retry_body_non_file_like(self):
2505+
req = Request("http://example.com/", b"abc")
2506+
self.assertTrue(_rewind_auth_retry_body(req))
2507+
2508+
def test_rewind_auth_retry_body(self):
2509+
data = io.BytesIO(b"xxxabc")
2510+
data.seek(3)
2511+
req = Request("http://example.com/", data)
2512+
req._body_start_pos = 3
2513+
data.read()
2514+
self.assertTrue(_rewind_auth_retry_body(req))
2515+
self.assertEqual(data.tell(), 3)
2516+
self.assertEqual(data.read(), b"abc")
2517+
2518+
def test_rewind_auth_retry_body_non_seekable(self):
2519+
class NonSeekableReader:
2520+
def read(self, n=-1):
2521+
return b"abc"
2522+
2523+
req = Request("http://example.com", NonSeekableReader())
2524+
self.assertFalse(_rewind_auth_retry_body(req))
2525+
2526+
def test_basic_auth_retry_file_body(self):
2527+
realm = "acme"
2528+
url = "http://example.com/"
2529+
body = b"basic auth retry file body"
2530+
auth_401 = (
2531+
b"HTTP/1.1 401 Unauthorized\r\n"
2532+
b'WWW-Authenticate: Basic realm="' + realm.encode() + b'"\r\n'
2533+
b"Connection: close\r\n\r\n"
2534+
)
2535+
auth_200 = b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n"
2536+
http_handler_cls, conn_cls = _make_auth_retry_http_handler([auth_401, auth_200])
2537+
2538+
password_manager = MockPasswordManager()
2539+
password_manager.add_password(realm, url, "user", "pass")
2540+
auth_handler = HTTPBasicAuthHandler(password_manager)
2541+
2542+
sent = _open_with_auth_retry(auth_handler, http_handler_cls, conn_cls, url, io.BytesIO(body))
2543+
groups = _group_sent_by_request(sent)
2544+
self.assertEqual(len(groups), 2)
2545+
self.assertEqual(_body_from_request_chunks(groups[0]), body)
2546+
self.assertEqual(_body_from_request_chunks(groups[1]), body)
2547+
2548+
def test_basic_auth_retry_file_body_from_nonzero_offset(self):
2549+
# The retry must resume from the position the body was in before
2550+
# the original request was sent, not from the start of the stream.
2551+
realm = "acme"
2552+
url = "http://example.com/"
2553+
prefix = b"SKIPME|"
2554+
body = b"basic auth retry file body"
2555+
auth_401 = (
2556+
b"HTTP/1.1 401 Unauthorized\r\n"
2557+
b'WWW-Authenticate: Basic realm="' + realm.encode() + b'"\r\n'
2558+
b"Connection: close\r\n\r\n"
2559+
)
2560+
auth_200 = b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n"
2561+
http_handler_cls, conn_cls = _make_auth_retry_http_handler([auth_401, auth_200])
2562+
2563+
password_manager = MockPasswordManager()
2564+
password_manager.add_password(realm, url, "user", "pass")
2565+
auth_handler = HTTPBasicAuthHandler(password_manager)
2566+
2567+
data = io.BytesIO(prefix + body)
2568+
data.seek(len(prefix))
2569+
sent = _open_with_auth_retry(auth_handler, http_handler_cls, conn_cls, url, data)
2570+
groups = _group_sent_by_request(sent)
2571+
self.assertEqual(len(groups), 2)
2572+
self.assertEqual(_body_from_request_chunks(groups[0]), body)
2573+
self.assertEqual(_body_from_request_chunks(groups[1]), body)
2574+
2575+
def test_basic_auth_retry_mmap_body(self):
2576+
realm = "acme"
2577+
url = "http://www.example.com/"
2578+
body = b"mmap auth retry file body"
2579+
auth_401 = (
2580+
b"HTTP/1.1 401 Unauthorized\r\n"
2581+
b'WWW-Authenticate: Basic realm="' + realm.encode() + b'"\r\n'
2582+
b"Connection: close\r\n\r\n"
2583+
)
2584+
auth_200 = b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n"
2585+
http_handler_cls, conn_cls = _make_auth_retry_http_handler([auth_401, auth_200])
2586+
password_manager = MockPasswordManager()
2587+
password_manager.add_password(realm, url, "user", "pass")
2588+
auth_handler = HTTPBasicAuthHandler(password_manager)
2589+
2590+
with tempfile.NamedTemporaryFile() as tmp:
2591+
tmp.write(body)
2592+
tmp.flush()
2593+
with mmap.mmap(tmp.fileno(), 0, access=mmap.ACCESS_READ) as mm:
2594+
sent = _open_with_auth_retry(auth_handler, http_handler_cls, conn_cls, url, mm)
2595+
2596+
groups = _group_sent_by_request(sent)
2597+
self.assertEqual(len(groups), 2)
2598+
self.assertEqual(_body_from_request_chunks(groups[0]), body)
2599+
self.assertEqual(_body_from_request_chunks(groups[1]), body)
2600+
2601+
def test_digest_auth_retry_file_body(self):
2602+
realm = "acme"
2603+
url = "http://example.com/"
2604+
body = b"digest auth retry file body"
2605+
auth_401 = (
2606+
b"HTTP/1.1 401 Unauthorized\r\n"
2607+
b'WWW-Authenticate: Digest realm="' + realm.encode() + b'", '
2608+
b'qop="auth", nonce="abc", opaque="def"\r\n'
2609+
b"Connection: close\r\n\r\n"
2610+
)
2611+
auth_200 = b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n"
2612+
http_handler_cls, conn_cls = _make_auth_retry_http_handler([auth_401, auth_200])
2613+
2614+
password_manager = MockPasswordManager()
2615+
password_manager.add_password(realm, url, "user", "pass")
2616+
auth_handler = urllib.request.HTTPDigestAuthHandler(password_manager)
2617+
2618+
sent = _open_with_auth_retry(auth_handler, http_handler_cls, conn_cls, url, io.BytesIO(body))
2619+
groups = _group_sent_by_request(sent)
2620+
self.assertEqual(len(groups), 2)
2621+
self.assertEqual(_body_from_request_chunks(groups[0]), body)
2622+
self.assertEqual(_body_from_request_chunks(groups[1]), body)
2623+
2624+
def test_basic_auth_retry_explicit_content_length(self):
2625+
realm = "acme"
2626+
url = "http://example.com/"
2627+
body = b"x" * 30
2628+
auth_401 = (
2629+
b"HTTP/1.1 401 Unauthorized\r\n"
2630+
b'WWW-Authenticate: Basic realm="' + realm.encode() + b'"\r\n'
2631+
b"Connection: close\r\n\r\n"
2632+
)
2633+
auth_200 = b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n"
2634+
http_handler_cls, conn_cls = _make_auth_retry_http_handler([auth_401, auth_200])
2635+
2636+
password_manager = MockPasswordManager()
2637+
password_manager.add_password(realm, url, "user", "pass")
2638+
auth_handler = HTTPBasicAuthHandler(password_manager)
2639+
2640+
conn_cls.sent_requests.clear()
2641+
conn_cls._request_count = 0
2642+
opener = OpenerDirector()
2643+
opener.add_handler(auth_handler)
2644+
opener.add_handler(http_handler_cls())
2645+
opener.add_handler(HTTPErrorProcessor())
2646+
opener.add_handler(HTTPDefaultErrorHandler())
2647+
2648+
req = Request(url, io.BytesIO(body), {"Content-Length": "30"})
2649+
opener.open(req)
2650+
sent = conn_cls.sent_requests
2651+
2652+
groups = _group_sent_by_request(sent)
2653+
self.assertEqual(len(groups), 2)
2654+
self.assertEqual(_body_from_request_chunks(groups[0]), body)
2655+
self.assertEqual(_body_from_request_chunks(groups[1]), body)
2656+
2657+
def test_basic_auth_retry_non_seekable_body(self):
2658+
realm = "acme"
2659+
url = "http://example.com/"
2660+
auth_401 = (
2661+
b"HTTP/1.1 401 Unauthorized\r\n"
2662+
b'WWW-Authenticate: Basic realm="' + realm.encode() + b'"\r\n'
2663+
b"Connection: close\r\n\r\n"
2664+
)
2665+
auth_200 = b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n"
2666+
http_handler_cls, conn_cls = _make_auth_retry_http_handler([auth_401, auth_200])
2667+
2668+
password_manager = MockPasswordManager()
2669+
password_manager.add_password(realm, url, "user", "pass")
2670+
auth_handler = HTTPBasicAuthHandler(password_manager)
2671+
2672+
class NonSeekableReader:
2673+
_data = b"pipe-data"
2674+
2675+
def read(self, n=-1):
2676+
if self._data is None:
2677+
return b""
2678+
chunk = self._data
2679+
self._data = None
2680+
return chunk
2681+
2682+
opener = OpenerDirector()
2683+
opener.add_handler(auth_handler)
2684+
opener.add_handler(http_handler_cls())
2685+
opener.add_handler(HTTPErrorProcessor())
2686+
opener.add_handler(HTTPDefaultErrorHandler())
2687+
2688+
req = Request(url, NonSeekableReader(), method="POST")
2689+
with self.assertRaises(urllib.error.HTTPError) as cm:
2690+
opener.open(req)
2691+
self.addCleanup(cm.exception.close)
2692+
self.assertEqual(cm.exception.code, 401)
2693+
# The retry was abandoned rather than resending a partial body.
2694+
self.assertEqual(len(_group_sent_by_request(conn_cls.sent_requests)), 1)
2695+
23912696
def test_http_closed(self):
23922697
"""Test the connection is cleaned up when the response is closed"""
23932698
for (transfer, data) in (

0 commit comments

Comments
 (0)