Skip to content

Commit bc35f90

Browse files
orsenthilmiguendes
andauthored
[3.14] gh-88037 - Move port validation logic to parsing time (#159122) (#161621)
gh-88037 - Move port validation logic to parsing time (#159122) * Move port validation logic to parsing time * Fix CI. Handle the special case of file://C:\\ parsed by url2pathname function where ":" should not be treated as port separator. --------- (cherry picked from commit 403f6dd) Co-authored-by: Miguel Mendes <miguel.mdebrito@gmail.com>
1 parent 8659ded commit bc35f90

6 files changed

Lines changed: 84 additions & 35 deletions

File tree

‎Lib/test/test_urllib.py‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import email.message
88
import io
99
import unittest
10+
import unittest.mock
1011
from test import support
1112
from test.support import os_helper
1213
from test.support import socket_helper
@@ -400,11 +401,17 @@ def test_url_host_with_newline_header_injection_rejected(self):
400401
host = "localhost\r\nX-injected: header\r\n"
401402
schemeless_url = "//" + host + ":8080/test/?test=a"
402403
try:
403-
InvalidURL = http.client.InvalidURL
404+
# Once \r\n are stripped from the URL, the ':' in
405+
# "X-injected:" is mistaken for the port separator, and
406+
# urlsplit()/urlparse() now reject the resulting
407+
# non-numeric "port" at parse time, before a Request is
408+
# even built -- rejecting the injection earlier than
409+
# http.client's own InvalidURL check ever gets a chance to.
404410
with self.assertRaisesRegex(
405-
InvalidURL, r"contain control.*\\r"):
411+
ValueError, "Port could not be cast to integer value"):
406412
urllib.request.urlopen(f"http:{schemeless_url}")
407-
with self.assertRaisesRegex(InvalidURL, r"contain control.*\\n"):
413+
with self.assertRaisesRegex(
414+
ValueError, "Port could not be cast to integer value"):
408415
urllib.request.urlopen(f"https:{schemeless_url}")
409416
finally:
410417
self.unfakehttp()
@@ -1672,6 +1679,17 @@ def test_url2pathname_resolve_host(self):
16721679
self.assertEqual(fn(f'//{socket.gethostname()}/foo/bar'), f'{sep}foo{sep}bar')
16731680
self.assertEqual(fn(f'//{socket.gethostname()}/foo/bar', resolve_host=True), f'{sep}foo{sep}bar')
16741681

1682+
def test_url2pathname_win_netloc_not_validated_as_port(self):
1683+
# The Windows branch of url2pathname() stuffs things like drive
1684+
# letters into the netloc (e.g. '//C:\\' -> authority 'C:\\'),
1685+
# which is not a real host[:port] pair. url2pathname() must not
1686+
# reject it as an invalid port; run the Windows branch directly
1687+
# so this is exercised on every platform, not just win32.
1688+
with unittest.mock.patch('os.name', 'nt'):
1689+
fn = urllib.request.url2pathname
1690+
self.assertEqual(fn('//C:\\'), 'C:\\')
1691+
self.assertEqual(fn('//C:80/'), 'C:80\\')
1692+
16751693
@unittest.skipUnless(sys.platform == 'win32',
16761694
'test specific to Windows pathnames.')
16771695
def test_url2pathname_win(self):

‎Lib/test/test_urllib2.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -898,9 +898,9 @@ def test_file(self):
898898
parsed._replace(netloc='localhost:80').geturl(),
899899
"file:///file_does_not_exist.txt",
900900
"file://not-a-local-host.com//dir/file.txt",
901-
"file://%s:80%s/%s" % (socket.gethostbyname('localhost'),
901+
"file://%s:80/%s/%s" % (socket.gethostbyname('localhost'),
902902
os.getcwd(), TESTFN),
903-
"file://somerandomhost.ontheinternet.com%s/%s" %
903+
"file://somerandomhost.ontheinternet.com/%s/%s" %
904904
(os.getcwd(), TESTFN),
905905
]:
906906
try:

‎Lib/test/test_urlparse.py‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -874,9 +874,8 @@ def test_urlsplit_attributes(self):
874874

875875
# Verify an illegal port raises ValueError
876876
url = b"HTTP://WWW.PYTHON.ORG:65536/doc/#frag"
877-
p = urllib.parse.urlsplit(url)
878877
with self.assertRaisesRegex(ValueError, "out of range"):
879-
p.port
878+
urllib.parse.urlsplit(url)
880879

881880
def test_urlsplit_remove_unsafe_bytes(self):
882881
# Remove ASCII tabs and newlines from input
@@ -986,10 +985,8 @@ def test_attributes_bad_port(self, bytes, parse, port):
986985
self.skipTest('non-ASCII bytes')
987986
netloc = str_encode(netloc)
988987
url = str_encode(url)
989-
p = parse(url)
990-
self.assertEqual(p.netloc, netloc)
991988
with self.assertRaises(ValueError):
992-
p.port
989+
parse(url)
993990

994991
@support.subTests('bytes', (False, True))
995992
@support.subTests('parse', (urllib.parse.urlsplit, urllib.parse.urlparse))
@@ -1443,13 +1440,11 @@ def test_splitting_bracketed_hosts(self):
14431440

14441441
def test_port_casting_failure_message(self):
14451442
message = "Port could not be cast to integer value as 'oracle'"
1446-
p1 = urllib.parse.urlparse('http://Server=sde; Service=sde:oracle')
14471443
with self.assertRaisesRegex(ValueError, message):
1448-
p1.port
1444+
urllib.parse.urlparse('http://Server=sde; Service=sde:oracle')
14491445

1450-
p2 = urllib.parse.urlsplit('http://Server=sde; Service=sde:oracle')
14511446
with self.assertRaisesRegex(ValueError, message):
1452-
p2.port
1447+
urllib.parse.urlsplit('http://Server=sde; Service=sde:oracle')
14531448

14541449
def test_telurl_params(self):
14551450
p1 = urllib.parse.urlparse('tel:123-4;phone-context=+1-650-516')

‎Lib/urllib/parse.py‎

Lines changed: 36 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,27 @@ def _coerce_args(*args):
132132
return args + (_noop,)
133133
return _decode_args(args) + (_encode_result,)
134134

135+
def _parse_hostinfo(netloc):
136+
_, _, hostinfo = netloc.rpartition('@')
137+
_, have_open_br, bracketed = hostinfo.partition('[')
138+
if have_open_br:
139+
hostname, _, port = bracketed.partition(']')
140+
_, _, port = port.partition(':')
141+
else:
142+
hostname, _, port = hostinfo.partition(':')
143+
if not port:
144+
port = None
145+
return hostname, port
146+
147+
def _validate_port(port):
148+
if port.isdigit() and port.isascii():
149+
port = int(port)
150+
else:
151+
raise ValueError(f"Port could not be cast to integer value as {port!r}")
152+
if not (0 <= port <= 65535):
153+
raise ValueError("Port out of range 0-65535")
154+
return port
155+
135156
# Result objects are more helpful than simple tuples
136157
class _ResultMixinStr(object):
137158
"""Standard approach to encoding parsed results from str to bytes"""
@@ -176,12 +197,7 @@ def hostname(self):
176197
def port(self):
177198
port = self._hostinfo[1]
178199
if port is not None:
179-
if port.isdigit() and port.isascii():
180-
port = int(port)
181-
else:
182-
raise ValueError(f"Port could not be cast to integer value as {port!r}")
183-
if not (0 <= port <= 65535):
184-
raise ValueError("Port out of range 0-65535")
200+
port = _validate_port(port)
185201
return port
186202

187203
__class_getitem__ = classmethod(types.GenericAlias)
@@ -205,16 +221,9 @@ def _userinfo(self):
205221
@property
206222
def _hostinfo(self):
207223
netloc = self.netloc
208-
_, _, hostinfo = netloc.rpartition('@')
209-
_, have_open_br, bracketed = hostinfo.partition('[')
210-
if have_open_br:
211-
hostname, _, port = bracketed.partition(']')
212-
_, _, port = port.partition(':')
213-
else:
214-
hostname, _, port = hostinfo.partition(':')
215-
if not port:
216-
port = None
217-
return hostname, port
224+
if netloc is None:
225+
return None, None
226+
return _parse_hostinfo(netloc)
218227

219228

220229
class _NetlocResultMixinBytes(_NetlocResultMixinBase, _ResultMixinBytes):
@@ -423,9 +432,7 @@ def _splitnetloc(url, start=0):
423432
delim = min(delim, wdelim) # use earliest delim position
424433
return url[start:delim], url[delim:] # return (domain, rest)
425434

426-
def _checknetloc(netloc):
427-
if not netloc or netloc.isascii():
428-
return
435+
def _checknetloc_nfkc(netloc):
429436
# looking for characters like \u2100 that expand to 'a/c'
430437
# IDNA uses NFKC equivalence, so normalize for this check
431438
import unicodedata
@@ -441,6 +448,16 @@ def _checknetloc(netloc):
441448
raise ValueError("netloc '" + netloc + "' contains invalid " +
442449
"characters under NFKC normalization")
443450

451+
def _checknetloc(netloc):
452+
if not netloc:
453+
return
454+
if not netloc.isascii():
455+
_checknetloc_nfkc(netloc)
456+
457+
_, port = _parse_hostinfo(netloc)
458+
if port is not None:
459+
_validate_port(port)
460+
444461
def _check_bracketed_netloc(netloc):
445462
# Note that this function must mirror the splitting
446463
# done in NetlocResultMixins._hostinfo().

‎Lib/urllib/request.py‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,8 @@
101101
from urllib.parse import (
102102
urlparse, urlsplit, urljoin, unwrap, quote, unquote,
103103
_splittype, _splithost, _splitport, _splituser, _splitpasswd,
104-
_splitattr, _splitvalue, _splittag,
104+
_splitattr, _splitvalue, _splittag, _splitnetloc,
105+
_WHATWG_C0_CONTROL_OR_SPACE, _UNSAFE_URL_BYTES_TO_REMOVE,
105106
unquote_to_bytes, urlunparse)
106107
from urllib.response import addinfourl, addclosehook
107108

@@ -1751,7 +1752,22 @@ def url2pathname(url, *, require_scheme=False, resolve_host=False):
17511752
"""
17521753
if not require_scheme:
17531754
url = 'file:' + url
1754-
scheme, authority, url = urlsplit(url)[:3] # Discard query and fragment.
1755+
# Match urlsplit()'s own control-character handling up front, so the
1756+
# except branch below can reuse this same cleaned-up string.
1757+
url = url.lstrip(_WHATWG_C0_CONTROL_OR_SPACE)
1758+
for b in _UNSAFE_URL_BYTES_TO_REMOVE:
1759+
url = url.replace(b, '')
1760+
try:
1761+
scheme, authority, url = urlsplit(url)[:3] # Discard query and fragment.
1762+
except ValueError:
1763+
# The authority here is not a real host[:port] pair -- it may
1764+
# hold a Windows drive letter (e.g. 'C:\\') -- so urlsplit()'s
1765+
# eager port check doesn't apply; split without it instead.
1766+
scheme, _, url = url.partition(':')
1767+
scheme = scheme.lower()
1768+
authority = ''
1769+
if url[:2] == '//':
1770+
authority, url = _splitnetloc(url, 2)
17551771
if scheme != 'file':
17561772
raise URLError("URL is missing a 'file:' scheme")
17571773
if os.name == 'nt':
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
:func:`urllib.parse.urlsplit` and :func:`urllib.parse.urlparse` now raise
2+
:exc:`ValueError` for an invalid port when the URL is parsed, instead of
3+
when the :attr:`!port` attribute is read. Patch by Miguel Brito and Senthil Kumaran.

0 commit comments

Comments
 (0)