|
9 | 9 | import os |
10 | 10 | import io |
11 | 11 | import hashlib |
| 12 | +import mmap |
12 | 13 | import ftplib |
13 | 14 | import socket |
14 | 15 | import array |
|
19 | 20 | import urllib.request |
20 | 21 | # The proxy bypass method imported below has logic specific to the |
21 | 22 | # corresponding system but is testable on all platforms. |
22 | | -from urllib.request import (Request, OpenerDirector, HTTPBasicAuthHandler, |
23 | | - HTTPPasswordMgrWithPriorAuth, _parse_proxy, |
24 | | - _proxy_bypass_winreg_override, |
25 | | - _proxy_bypass_macosx_sysconf, |
| 23 | +from urllib.request import (HTTPDefaultErrorHandler, HTTPErrorProcessor, |
| 24 | + Request, OpenerDirector, HTTPBasicAuthHandler, |
| 25 | + HTTPPasswordMgrWithPriorAuth, |
| 26 | + _parse_proxy, _proxy_bypass_winreg_override, |
| 27 | + _proxy_bypass_macosx_sysconf, _rewind_auth_retry_body, |
26 | 28 | AbstractDigestAuthHandler) |
27 | 29 | from urllib.parse import urlsplit |
28 | 30 | import urllib.error |
@@ -672,6 +674,113 @@ def http_open(self, req): |
672 | 674 | return MockResponse(self.code, name, MockFile(), "", req.get_full_url()) |
673 | 675 |
|
674 | 676 |
|
| 677 | +class _AuthRetrySocket: |
| 678 | + def __init__(self, response, sent_requests): |
| 679 | + self._response = io.BytesIO(response) |
| 680 | + self._sent_requests = sent_requests |
| 681 | + self._closed = False |
| 682 | + |
| 683 | + def sendall(self, data): |
| 684 | + self._sent_requests.append(data) |
| 685 | + |
| 686 | + def makefile(self, mode='r', buffering=None, *, encoding=None, errors=None): |
| 687 | + return self |
| 688 | + |
| 689 | + def read(self, amt=-1): |
| 690 | + if self._closed: |
| 691 | + return b"" |
| 692 | + return self._response.read(amt) |
| 693 | + |
| 694 | + def readline(self, length=-1): |
| 695 | + if self._closed: |
| 696 | + return b"" |
| 697 | + return self._response.readline(length) |
| 698 | + |
| 699 | + def flush(self): |
| 700 | + self._response.flush() |
| 701 | + |
| 702 | + def close(self): |
| 703 | + self._closed = True |
| 704 | + |
| 705 | +def _open_with_auth_retry(auth_handler, http_handler_cls, conn_cls, url, data): |
| 706 | + conn_cls.sent_requests.clear() |
| 707 | + conn_cls._request_count = 0 |
| 708 | + opener = OpenerDirector() |
| 709 | + opener.add_handler(auth_handler) |
| 710 | + opener.add_handler(http_handler_cls()) |
| 711 | + opener.add_handler(HTTPErrorProcessor()) |
| 712 | + opener.add_handler(HTTPDefaultErrorHandler()) |
| 713 | + opener.open(Request(url, data, method="POST")) |
| 714 | + return conn_cls.sent_requests |
| 715 | + |
| 716 | +def _decode_chunked_http_body(raw): |
| 717 | + body = raw |
| 718 | + result = b"" |
| 719 | + while body: |
| 720 | + size_line, sep, body = body.partition(b"\r\n") |
| 721 | + if not sep: |
| 722 | + break |
| 723 | + size = int(size_line.split(b";", 1)[0], 16) |
| 724 | + if size == 0: |
| 725 | + break |
| 726 | + result += body[:size] |
| 727 | + body = body[size+2:] |
| 728 | + return result |
| 729 | + |
| 730 | +def _group_sent_by_request(sent): |
| 731 | + groups = [] |
| 732 | + current = [] |
| 733 | + for chunk in sent: |
| 734 | + if chunk.startswith(b"POST ") and current: |
| 735 | + groups.append(current) |
| 736 | + current = [chunk] |
| 737 | + else: |
| 738 | + current.append(chunk) |
| 739 | + |
| 740 | + if current: |
| 741 | + groups.append(current) |
| 742 | + return groups |
| 743 | + |
| 744 | +def _body_from_request_chunks(chunks): |
| 745 | + return _body_from_sent_http_request(b"".join(chunks)) |
| 746 | + |
| 747 | +def _body_from_sent_http_request(raw): |
| 748 | + _, sep, body = raw.partition(b"\r\n\r\n") |
| 749 | + if not sep: |
| 750 | + return b"" |
| 751 | + header_block = raw[:raw.index(sep)].lower() |
| 752 | + if b"transfer-encoding: chunked" in header_block: |
| 753 | + return _decode_chunked_http_body(body) |
| 754 | + if b"content-length: 0" in header_block: |
| 755 | + return b"" |
| 756 | + return body.split(b"\r\n", 1)[0] |
| 757 | + |
| 758 | +def _make_auth_retry_http_connection(responses): |
| 759 | + class AuthRetryHTTPConnection(http.client.HTTPConnection): |
| 760 | + sent_requests = [] |
| 761 | + _request_count = 0 |
| 762 | + |
| 763 | + def connect(self): |
| 764 | + request_index = type(self)._request_count |
| 765 | + type(self)._request_count += 1 |
| 766 | + response = responses[request_index] |
| 767 | + self.sock = _AuthRetrySocket(response, type(self).sent_requests) |
| 768 | + |
| 769 | + return AuthRetryHTTPConnection |
| 770 | + |
| 771 | +def _make_auth_retry_http_handler(responses): |
| 772 | + conn_cls = _make_auth_retry_http_connection(responses) |
| 773 | + |
| 774 | + class AuthRetryHTTPHandler(urllib.request.HTTPHandler): |
| 775 | + def __init__(self): |
| 776 | + super().__init__() |
| 777 | + self.httpconn = conn_cls |
| 778 | + |
| 779 | + def http_open(self, req): |
| 780 | + return self.do_open(self.httpconn, req) |
| 781 | + |
| 782 | + return AuthRetryHTTPHandler, conn_cls |
| 783 | + |
675 | 784 |
|
676 | 785 | class MockPasswordManager: |
677 | 786 | def add_password(self, realm, uri, user, password): |
@@ -2388,6 +2497,202 @@ def test_basic_prior_auth_send_after_first_success(self): |
2388 | 2497 | # expect request to be sent with auth header |
2389 | 2498 | self.assertTrue(http_handler.has_auth_header) |
2390 | 2499 |
|
| 2500 | + def test_rewind_auth_retry_body_no_data(self): |
| 2501 | + req = Request("http://example.com/") |
| 2502 | + self.assertTrue(_rewind_auth_retry_body(req)) |
| 2503 | + |
| 2504 | + def test_rewind_auth_retry_body_non_file_like(self): |
| 2505 | + req = Request("http://example.com/", b"abc") |
| 2506 | + self.assertTrue(_rewind_auth_retry_body(req)) |
| 2507 | + |
| 2508 | + def test_rewind_auth_retry_body(self): |
| 2509 | + data = io.BytesIO(b"xxxabc") |
| 2510 | + data.seek(3) |
| 2511 | + req = Request("http://example.com/", data) |
| 2512 | + req._body_start_pos = 3 |
| 2513 | + data.read() |
| 2514 | + self.assertTrue(_rewind_auth_retry_body(req)) |
| 2515 | + self.assertEqual(data.tell(), 3) |
| 2516 | + self.assertEqual(data.read(), b"abc") |
| 2517 | + |
| 2518 | + def test_rewind_auth_retry_body_non_seekable(self): |
| 2519 | + class NonSeekableReader: |
| 2520 | + def read(self, n=-1): |
| 2521 | + return b"abc" |
| 2522 | + |
| 2523 | + req = Request("http://example.com", NonSeekableReader()) |
| 2524 | + self.assertFalse(_rewind_auth_retry_body(req)) |
| 2525 | + |
| 2526 | + def test_basic_auth_retry_file_body(self): |
| 2527 | + realm = "acme" |
| 2528 | + url = "http://example.com/" |
| 2529 | + body = b"basic auth retry file body" |
| 2530 | + auth_401 = ( |
| 2531 | + b"HTTP/1.1 401 Unauthorized\r\n" |
| 2532 | + b'WWW-Authenticate: Basic realm="' + realm.encode() + b'"\r\n' |
| 2533 | + b"Connection: close\r\n\r\n" |
| 2534 | + ) |
| 2535 | + auth_200 = b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n" |
| 2536 | + http_handler_cls, conn_cls = _make_auth_retry_http_handler([auth_401, auth_200]) |
| 2537 | + |
| 2538 | + password_manager = MockPasswordManager() |
| 2539 | + password_manager.add_password(realm, url, "user", "pass") |
| 2540 | + auth_handler = HTTPBasicAuthHandler(password_manager) |
| 2541 | + |
| 2542 | + sent = _open_with_auth_retry(auth_handler, http_handler_cls, conn_cls, url, io.BytesIO(body)) |
| 2543 | + groups = _group_sent_by_request(sent) |
| 2544 | + self.assertEqual(len(groups), 2) |
| 2545 | + self.assertEqual(_body_from_request_chunks(groups[0]), body) |
| 2546 | + self.assertEqual(_body_from_request_chunks(groups[1]), body) |
| 2547 | + |
| 2548 | + def test_basic_auth_retry_file_body_from_nonzero_offset(self): |
| 2549 | + # The retry must resume from the position the body was in before |
| 2550 | + # the original request was sent, not from the start of the stream. |
| 2551 | + realm = "acme" |
| 2552 | + url = "http://example.com/" |
| 2553 | + prefix = b"SKIPME|" |
| 2554 | + body = b"basic auth retry file body" |
| 2555 | + auth_401 = ( |
| 2556 | + b"HTTP/1.1 401 Unauthorized\r\n" |
| 2557 | + b'WWW-Authenticate: Basic realm="' + realm.encode() + b'"\r\n' |
| 2558 | + b"Connection: close\r\n\r\n" |
| 2559 | + ) |
| 2560 | + auth_200 = b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n" |
| 2561 | + http_handler_cls, conn_cls = _make_auth_retry_http_handler([auth_401, auth_200]) |
| 2562 | + |
| 2563 | + password_manager = MockPasswordManager() |
| 2564 | + password_manager.add_password(realm, url, "user", "pass") |
| 2565 | + auth_handler = HTTPBasicAuthHandler(password_manager) |
| 2566 | + |
| 2567 | + data = io.BytesIO(prefix + body) |
| 2568 | + data.seek(len(prefix)) |
| 2569 | + sent = _open_with_auth_retry(auth_handler, http_handler_cls, conn_cls, url, data) |
| 2570 | + groups = _group_sent_by_request(sent) |
| 2571 | + self.assertEqual(len(groups), 2) |
| 2572 | + self.assertEqual(_body_from_request_chunks(groups[0]), body) |
| 2573 | + self.assertEqual(_body_from_request_chunks(groups[1]), body) |
| 2574 | + |
| 2575 | + def test_basic_auth_retry_mmap_body(self): |
| 2576 | + realm = "acme" |
| 2577 | + url = "http://www.example.com/" |
| 2578 | + body = b"mmap auth retry file body" |
| 2579 | + auth_401 = ( |
| 2580 | + b"HTTP/1.1 401 Unauthorized\r\n" |
| 2581 | + b'WWW-Authenticate: Basic realm="' + realm.encode() + b'"\r\n' |
| 2582 | + b"Connection: close\r\n\r\n" |
| 2583 | + ) |
| 2584 | + auth_200 = b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n" |
| 2585 | + http_handler_cls, conn_cls = _make_auth_retry_http_handler([auth_401, auth_200]) |
| 2586 | + password_manager = MockPasswordManager() |
| 2587 | + password_manager.add_password(realm, url, "user", "pass") |
| 2588 | + auth_handler = HTTPBasicAuthHandler(password_manager) |
| 2589 | + |
| 2590 | + with tempfile.NamedTemporaryFile() as tmp: |
| 2591 | + tmp.write(body) |
| 2592 | + tmp.flush() |
| 2593 | + with mmap.mmap(tmp.fileno(), 0, access=mmap.ACCESS_READ) as mm: |
| 2594 | + sent = _open_with_auth_retry(auth_handler, http_handler_cls, conn_cls, url, mm) |
| 2595 | + |
| 2596 | + groups = _group_sent_by_request(sent) |
| 2597 | + self.assertEqual(len(groups), 2) |
| 2598 | + self.assertEqual(_body_from_request_chunks(groups[0]), body) |
| 2599 | + self.assertEqual(_body_from_request_chunks(groups[1]), body) |
| 2600 | + |
| 2601 | + def test_digest_auth_retry_file_body(self): |
| 2602 | + realm = "acme" |
| 2603 | + url = "http://example.com/" |
| 2604 | + body = b"digest auth retry file body" |
| 2605 | + auth_401 = ( |
| 2606 | + b"HTTP/1.1 401 Unauthorized\r\n" |
| 2607 | + b'WWW-Authenticate: Digest realm="' + realm.encode() + b'", ' |
| 2608 | + b'qop="auth", nonce="abc", opaque="def"\r\n' |
| 2609 | + b"Connection: close\r\n\r\n" |
| 2610 | + ) |
| 2611 | + auth_200 = b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n" |
| 2612 | + http_handler_cls, conn_cls = _make_auth_retry_http_handler([auth_401, auth_200]) |
| 2613 | + |
| 2614 | + password_manager = MockPasswordManager() |
| 2615 | + password_manager.add_password(realm, url, "user", "pass") |
| 2616 | + auth_handler = urllib.request.HTTPDigestAuthHandler(password_manager) |
| 2617 | + |
| 2618 | + sent = _open_with_auth_retry(auth_handler, http_handler_cls, conn_cls, url, io.BytesIO(body)) |
| 2619 | + groups = _group_sent_by_request(sent) |
| 2620 | + self.assertEqual(len(groups), 2) |
| 2621 | + self.assertEqual(_body_from_request_chunks(groups[0]), body) |
| 2622 | + self.assertEqual(_body_from_request_chunks(groups[1]), body) |
| 2623 | + |
| 2624 | + def test_basic_auth_retry_explicit_content_length(self): |
| 2625 | + realm = "acme" |
| 2626 | + url = "http://example.com/" |
| 2627 | + body = b"x" * 30 |
| 2628 | + auth_401 = ( |
| 2629 | + b"HTTP/1.1 401 Unauthorized\r\n" |
| 2630 | + b'WWW-Authenticate: Basic realm="' + realm.encode() + b'"\r\n' |
| 2631 | + b"Connection: close\r\n\r\n" |
| 2632 | + ) |
| 2633 | + auth_200 = b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n" |
| 2634 | + http_handler_cls, conn_cls = _make_auth_retry_http_handler([auth_401, auth_200]) |
| 2635 | + |
| 2636 | + password_manager = MockPasswordManager() |
| 2637 | + password_manager.add_password(realm, url, "user", "pass") |
| 2638 | + auth_handler = HTTPBasicAuthHandler(password_manager) |
| 2639 | + |
| 2640 | + conn_cls.sent_requests.clear() |
| 2641 | + conn_cls._request_count = 0 |
| 2642 | + opener = OpenerDirector() |
| 2643 | + opener.add_handler(auth_handler) |
| 2644 | + opener.add_handler(http_handler_cls()) |
| 2645 | + opener.add_handler(HTTPErrorProcessor()) |
| 2646 | + opener.add_handler(HTTPDefaultErrorHandler()) |
| 2647 | + |
| 2648 | + req = Request(url, io.BytesIO(body), {"Content-Length": "30"}) |
| 2649 | + opener.open(req) |
| 2650 | + sent = conn_cls.sent_requests |
| 2651 | + |
| 2652 | + groups = _group_sent_by_request(sent) |
| 2653 | + self.assertEqual(len(groups), 2) |
| 2654 | + self.assertEqual(_body_from_request_chunks(groups[0]), body) |
| 2655 | + self.assertEqual(_body_from_request_chunks(groups[1]), body) |
| 2656 | + |
| 2657 | + def test_basic_auth_retry_non_seekable_body(self): |
| 2658 | + realm = "acme" |
| 2659 | + url = "http://example.com/" |
| 2660 | + auth_401 = ( |
| 2661 | + b"HTTP/1.1 401 Unauthorized\r\n" |
| 2662 | + b'WWW-Authenticate: Basic realm="' + realm.encode() + b'"\r\n' |
| 2663 | + b"Connection: close\r\n\r\n" |
| 2664 | + ) |
| 2665 | + auth_200 = b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n" |
| 2666 | + http_handler_cls, conn_cls = _make_auth_retry_http_handler([auth_401, auth_200]) |
| 2667 | + |
| 2668 | + password_manager = MockPasswordManager() |
| 2669 | + password_manager.add_password(realm, url, "user", "pass") |
| 2670 | + auth_handler = HTTPBasicAuthHandler(password_manager) |
| 2671 | + |
| 2672 | + class NonSeekableReader: |
| 2673 | + _data = b"pipe-data" |
| 2674 | + |
| 2675 | + def read(self, n=-1): |
| 2676 | + if self._data is None: |
| 2677 | + return b"" |
| 2678 | + chunk = self._data |
| 2679 | + self._data = None |
| 2680 | + return chunk |
| 2681 | + |
| 2682 | + opener = OpenerDirector() |
| 2683 | + opener.add_handler(auth_handler) |
| 2684 | + opener.add_handler(http_handler_cls()) |
| 2685 | + opener.add_handler(HTTPErrorProcessor()) |
| 2686 | + opener.add_handler(HTTPDefaultErrorHandler()) |
| 2687 | + |
| 2688 | + req = Request(url, NonSeekableReader(), method="POST") |
| 2689 | + with self.assertRaises(urllib.error.HTTPError) as cm: |
| 2690 | + opener.open(req) |
| 2691 | + self.addCleanup(cm.exception.close) |
| 2692 | + self.assertEqual(cm.exception.code, 401) |
| 2693 | + # The retry was abandoned rather than resending a partial body. |
| 2694 | + self.assertEqual(len(_group_sent_by_request(conn_cls.sent_requests)), 1) |
| 2695 | + |
2391 | 2696 | def test_http_closed(self): |
2392 | 2697 | """Test the connection is cleaned up when the response is closed""" |
2393 | 2698 | for (transfer, data) in ( |
|
0 commit comments