kopia lustrzana https://github.com/micropython/micropython-lib
urllib.parse: Add pristine from CPython 3.3.3.
Paul Sokolovsky
rodzic
3df5927825
commit
23538a2222
|
|
@ -0,0 +1,855 @@
|
|||
#! /usr/bin/env python3
|
||||
|
||||
from test import support
|
||||
import unittest
|
||||
import urllib.parse
|
||||
|
||||
RFC1808_BASE = "http://a/b/c/d;p?q#f"
|
||||
RFC2396_BASE = "http://a/b/c/d;p?q"
|
||||
RFC3986_BASE = 'http://a/b/c/d;p?q'
|
||||
SIMPLE_BASE = 'http://a/b/c/d'
|
||||
|
||||
# A list of test cases. Each test case is a two-tuple that contains
|
||||
# a string with the query and a dictionary with the expected result.
|
||||
|
||||
parse_qsl_test_cases = [
|
||||
("", []),
|
||||
("&", []),
|
||||
("&&", []),
|
||||
("=", [('', '')]),
|
||||
("=a", [('', 'a')]),
|
||||
("a", [('a', '')]),
|
||||
("a=", [('a', '')]),
|
||||
("a=", [('a', '')]),
|
||||
("&a=b", [('a', 'b')]),
|
||||
("a=a+b&b=b+c", [('a', 'a b'), ('b', 'b c')]),
|
||||
("a=1&a=2", [('a', '1'), ('a', '2')]),
|
||||
(b"", []),
|
||||
(b"&", []),
|
||||
(b"&&", []),
|
||||
(b"=", [(b'', b'')]),
|
||||
(b"=a", [(b'', b'a')]),
|
||||
(b"a", [(b'a', b'')]),
|
||||
(b"a=", [(b'a', b'')]),
|
||||
(b"a=", [(b'a', b'')]),
|
||||
(b"&a=b", [(b'a', b'b')]),
|
||||
(b"a=a+b&b=b+c", [(b'a', b'a b'), (b'b', b'b c')]),
|
||||
(b"a=1&a=2", [(b'a', b'1'), (b'a', b'2')]),
|
||||
]
|
||||
|
||||
class UrlParseTestCase(unittest.TestCase):
|
||||
|
||||
def checkRoundtrips(self, url, parsed, split):
|
||||
result = urllib.parse.urlparse(url)
|
||||
self.assertEqual(result, parsed)
|
||||
t = (result.scheme, result.netloc, result.path,
|
||||
result.params, result.query, result.fragment)
|
||||
self.assertEqual(t, parsed)
|
||||
# put it back together and it should be the same
|
||||
result2 = urllib.parse.urlunparse(result)
|
||||
self.assertEqual(result2, url)
|
||||
self.assertEqual(result2, result.geturl())
|
||||
|
||||
# the result of geturl() is a fixpoint; we can always parse it
|
||||
# again to get the same result:
|
||||
result3 = urllib.parse.urlparse(result.geturl())
|
||||
self.assertEqual(result3.geturl(), result.geturl())
|
||||
self.assertEqual(result3, result)
|
||||
self.assertEqual(result3.scheme, result.scheme)
|
||||
self.assertEqual(result3.netloc, result.netloc)
|
||||
self.assertEqual(result3.path, result.path)
|
||||
self.assertEqual(result3.params, result.params)
|
||||
self.assertEqual(result3.query, result.query)
|
||||
self.assertEqual(result3.fragment, result.fragment)
|
||||
self.assertEqual(result3.username, result.username)
|
||||
self.assertEqual(result3.password, result.password)
|
||||
self.assertEqual(result3.hostname, result.hostname)
|
||||
self.assertEqual(result3.port, result.port)
|
||||
|
||||
# check the roundtrip using urlsplit() as well
|
||||
result = urllib.parse.urlsplit(url)
|
||||
self.assertEqual(result, split)
|
||||
t = (result.scheme, result.netloc, result.path,
|
||||
result.query, result.fragment)
|
||||
self.assertEqual(t, split)
|
||||
result2 = urllib.parse.urlunsplit(result)
|
||||
self.assertEqual(result2, url)
|
||||
self.assertEqual(result2, result.geturl())
|
||||
|
||||
# check the fixpoint property of re-parsing the result of geturl()
|
||||
result3 = urllib.parse.urlsplit(result.geturl())
|
||||
self.assertEqual(result3.geturl(), result.geturl())
|
||||
self.assertEqual(result3, result)
|
||||
self.assertEqual(result3.scheme, result.scheme)
|
||||
self.assertEqual(result3.netloc, result.netloc)
|
||||
self.assertEqual(result3.path, result.path)
|
||||
self.assertEqual(result3.query, result.query)
|
||||
self.assertEqual(result3.fragment, result.fragment)
|
||||
self.assertEqual(result3.username, result.username)
|
||||
self.assertEqual(result3.password, result.password)
|
||||
self.assertEqual(result3.hostname, result.hostname)
|
||||
self.assertEqual(result3.port, result.port)
|
||||
|
||||
def test_qsl(self):
|
||||
for orig, expect in parse_qsl_test_cases:
|
||||
result = urllib.parse.parse_qsl(orig, keep_blank_values=True)
|
||||
self.assertEqual(result, expect, "Error parsing %r" % orig)
|
||||
expect_without_blanks = [v for v in expect if len(v[1])]
|
||||
result = urllib.parse.parse_qsl(orig, keep_blank_values=False)
|
||||
self.assertEqual(result, expect_without_blanks,
|
||||
"Error parsing %r" % orig)
|
||||
|
||||
def test_roundtrips(self):
|
||||
str_cases = [
|
||||
('file:///tmp/junk.txt',
|
||||
('file', '', '/tmp/junk.txt', '', '', ''),
|
||||
('file', '', '/tmp/junk.txt', '', '')),
|
||||
('imap://mail.python.org/mbox1',
|
||||
('imap', 'mail.python.org', '/mbox1', '', '', ''),
|
||||
('imap', 'mail.python.org', '/mbox1', '', '')),
|
||||
('mms://wms.sys.hinet.net/cts/Drama/09006251100.asf',
|
||||
('mms', 'wms.sys.hinet.net', '/cts/Drama/09006251100.asf',
|
||||
'', '', ''),
|
||||
('mms', 'wms.sys.hinet.net', '/cts/Drama/09006251100.asf',
|
||||
'', '')),
|
||||
('nfs://server/path/to/file.txt',
|
||||
('nfs', 'server', '/path/to/file.txt', '', '', ''),
|
||||
('nfs', 'server', '/path/to/file.txt', '', '')),
|
||||
('svn+ssh://svn.zope.org/repos/main/ZConfig/trunk/',
|
||||
('svn+ssh', 'svn.zope.org', '/repos/main/ZConfig/trunk/',
|
||||
'', '', ''),
|
||||
('svn+ssh', 'svn.zope.org', '/repos/main/ZConfig/trunk/',
|
||||
'', '')),
|
||||
('git+ssh://git@github.com/user/project.git',
|
||||
('git+ssh', 'git@github.com','/user/project.git',
|
||||
'','',''),
|
||||
('git+ssh', 'git@github.com','/user/project.git',
|
||||
'', '')),
|
||||
]
|
||||
def _encode(t):
|
||||
return (t[0].encode('ascii'),
|
||||
tuple(x.encode('ascii') for x in t[1]),
|
||||
tuple(x.encode('ascii') for x in t[2]))
|
||||
bytes_cases = [_encode(x) for x in str_cases]
|
||||
for url, parsed, split in str_cases + bytes_cases:
|
||||
self.checkRoundtrips(url, parsed, split)
|
||||
|
||||
def test_http_roundtrips(self):
|
||||
# urllib.parse.urlsplit treats 'http:' as an optimized special case,
|
||||
# so we test both 'http:' and 'https:' in all the following.
|
||||
# Three cheers for white box knowledge!
|
||||
str_cases = [
|
||||
('://www.python.org',
|
||||
('www.python.org', '', '', '', ''),
|
||||
('www.python.org', '', '', '')),
|
||||
('://www.python.org#abc',
|
||||
('www.python.org', '', '', '', 'abc'),
|
||||
('www.python.org', '', '', 'abc')),
|
||||
('://www.python.org?q=abc',
|
||||
('www.python.org', '', '', 'q=abc', ''),
|
||||
('www.python.org', '', 'q=abc', '')),
|
||||
('://www.python.org/#abc',
|
||||
('www.python.org', '/', '', '', 'abc'),
|
||||
('www.python.org', '/', '', 'abc')),
|
||||
('://a/b/c/d;p?q#f',
|
||||
('a', '/b/c/d', 'p', 'q', 'f'),
|
||||
('a', '/b/c/d;p', 'q', 'f')),
|
||||
]
|
||||
def _encode(t):
|
||||
return (t[0].encode('ascii'),
|
||||
tuple(x.encode('ascii') for x in t[1]),
|
||||
tuple(x.encode('ascii') for x in t[2]))
|
||||
bytes_cases = [_encode(x) for x in str_cases]
|
||||
str_schemes = ('http', 'https')
|
||||
bytes_schemes = (b'http', b'https')
|
||||
str_tests = str_schemes, str_cases
|
||||
bytes_tests = bytes_schemes, bytes_cases
|
||||
for schemes, test_cases in (str_tests, bytes_tests):
|
||||
for scheme in schemes:
|
||||
for url, parsed, split in test_cases:
|
||||
url = scheme + url
|
||||
parsed = (scheme,) + parsed
|
||||
split = (scheme,) + split
|
||||
self.checkRoundtrips(url, parsed, split)
|
||||
|
||||
def checkJoin(self, base, relurl, expected):
|
||||
str_components = (base, relurl, expected)
|
||||
self.assertEqual(urllib.parse.urljoin(base, relurl), expected)
|
||||
bytes_components = baseb, relurlb, expectedb = [
|
||||
x.encode('ascii') for x in str_components]
|
||||
self.assertEqual(urllib.parse.urljoin(baseb, relurlb), expectedb)
|
||||
|
||||
def test_unparse_parse(self):
|
||||
str_cases = ['Python', './Python','x-newscheme://foo.com/stuff','x://y','x:/y','x:/','/',]
|
||||
bytes_cases = [x.encode('ascii') for x in str_cases]
|
||||
for u in str_cases + bytes_cases:
|
||||
self.assertEqual(urllib.parse.urlunsplit(urllib.parse.urlsplit(u)), u)
|
||||
self.assertEqual(urllib.parse.urlunparse(urllib.parse.urlparse(u)), u)
|
||||
|
||||
def test_RFC1808(self):
|
||||
# "normal" cases from RFC 1808:
|
||||
self.checkJoin(RFC1808_BASE, 'g:h', 'g:h')
|
||||
self.checkJoin(RFC1808_BASE, 'g', 'http://a/b/c/g')
|
||||
self.checkJoin(RFC1808_BASE, './g', 'http://a/b/c/g')
|
||||
self.checkJoin(RFC1808_BASE, 'g/', 'http://a/b/c/g/')
|
||||
self.checkJoin(RFC1808_BASE, '/g', 'http://a/g')
|
||||
self.checkJoin(RFC1808_BASE, '//g', 'http://g')
|
||||
self.checkJoin(RFC1808_BASE, 'g?y', 'http://a/b/c/g?y')
|
||||
self.checkJoin(RFC1808_BASE, 'g?y/./x', 'http://a/b/c/g?y/./x')
|
||||
self.checkJoin(RFC1808_BASE, '#s', 'http://a/b/c/d;p?q#s')
|
||||
self.checkJoin(RFC1808_BASE, 'g#s', 'http://a/b/c/g#s')
|
||||
self.checkJoin(RFC1808_BASE, 'g#s/./x', 'http://a/b/c/g#s/./x')
|
||||
self.checkJoin(RFC1808_BASE, 'g?y#s', 'http://a/b/c/g?y#s')
|
||||
self.checkJoin(RFC1808_BASE, 'g;x', 'http://a/b/c/g;x')
|
||||
self.checkJoin(RFC1808_BASE, 'g;x?y#s', 'http://a/b/c/g;x?y#s')
|
||||
self.checkJoin(RFC1808_BASE, '.', 'http://a/b/c/')
|
||||
self.checkJoin(RFC1808_BASE, './', 'http://a/b/c/')
|
||||
self.checkJoin(RFC1808_BASE, '..', 'http://a/b/')
|
||||
self.checkJoin(RFC1808_BASE, '../', 'http://a/b/')
|
||||
self.checkJoin(RFC1808_BASE, '../g', 'http://a/b/g')
|
||||
self.checkJoin(RFC1808_BASE, '../..', 'http://a/')
|
||||
self.checkJoin(RFC1808_BASE, '../../', 'http://a/')
|
||||
self.checkJoin(RFC1808_BASE, '../../g', 'http://a/g')
|
||||
|
||||
# "abnormal" cases from RFC 1808:
|
||||
self.checkJoin(RFC1808_BASE, '', 'http://a/b/c/d;p?q#f')
|
||||
self.checkJoin(RFC1808_BASE, '../../../g', 'http://a/../g')
|
||||
self.checkJoin(RFC1808_BASE, '../../../../g', 'http://a/../../g')
|
||||
self.checkJoin(RFC1808_BASE, '/./g', 'http://a/./g')
|
||||
self.checkJoin(RFC1808_BASE, '/../g', 'http://a/../g')
|
||||
self.checkJoin(RFC1808_BASE, 'g.', 'http://a/b/c/g.')
|
||||
self.checkJoin(RFC1808_BASE, '.g', 'http://a/b/c/.g')
|
||||
self.checkJoin(RFC1808_BASE, 'g..', 'http://a/b/c/g..')
|
||||
self.checkJoin(RFC1808_BASE, '..g', 'http://a/b/c/..g')
|
||||
self.checkJoin(RFC1808_BASE, './../g', 'http://a/b/g')
|
||||
self.checkJoin(RFC1808_BASE, './g/.', 'http://a/b/c/g/')
|
||||
self.checkJoin(RFC1808_BASE, 'g/./h', 'http://a/b/c/g/h')
|
||||
self.checkJoin(RFC1808_BASE, 'g/../h', 'http://a/b/c/h')
|
||||
|
||||
# RFC 1808 and RFC 1630 disagree on these (according to RFC 1808),
|
||||
# so we'll not actually run these tests (which expect 1808 behavior).
|
||||
#self.checkJoin(RFC1808_BASE, 'http:g', 'http:g')
|
||||
#self.checkJoin(RFC1808_BASE, 'http:', 'http:')
|
||||
|
||||
def test_RFC2368(self):
|
||||
# Issue 11467: path that starts with a number is not parsed correctly
|
||||
self.assertEqual(urllib.parse.urlparse('mailto:1337@example.org'),
|
||||
('mailto', '', '1337@example.org', '', '', ''))
|
||||
|
||||
def test_RFC2396(self):
|
||||
# cases from RFC 2396
|
||||
|
||||
|
||||
self.checkJoin(RFC2396_BASE, 'g:h', 'g:h')
|
||||
self.checkJoin(RFC2396_BASE, 'g', 'http://a/b/c/g')
|
||||
self.checkJoin(RFC2396_BASE, './g', 'http://a/b/c/g')
|
||||
self.checkJoin(RFC2396_BASE, 'g/', 'http://a/b/c/g/')
|
||||
self.checkJoin(RFC2396_BASE, '/g', 'http://a/g')
|
||||
self.checkJoin(RFC2396_BASE, '//g', 'http://g')
|
||||
self.checkJoin(RFC2396_BASE, 'g?y', 'http://a/b/c/g?y')
|
||||
self.checkJoin(RFC2396_BASE, '#s', 'http://a/b/c/d;p?q#s')
|
||||
self.checkJoin(RFC2396_BASE, 'g#s', 'http://a/b/c/g#s')
|
||||
self.checkJoin(RFC2396_BASE, 'g?y#s', 'http://a/b/c/g?y#s')
|
||||
self.checkJoin(RFC2396_BASE, 'g;x', 'http://a/b/c/g;x')
|
||||
self.checkJoin(RFC2396_BASE, 'g;x?y#s', 'http://a/b/c/g;x?y#s')
|
||||
self.checkJoin(RFC2396_BASE, '.', 'http://a/b/c/')
|
||||
self.checkJoin(RFC2396_BASE, './', 'http://a/b/c/')
|
||||
self.checkJoin(RFC2396_BASE, '..', 'http://a/b/')
|
||||
self.checkJoin(RFC2396_BASE, '../', 'http://a/b/')
|
||||
self.checkJoin(RFC2396_BASE, '../g', 'http://a/b/g')
|
||||
self.checkJoin(RFC2396_BASE, '../..', 'http://a/')
|
||||
self.checkJoin(RFC2396_BASE, '../../', 'http://a/')
|
||||
self.checkJoin(RFC2396_BASE, '../../g', 'http://a/g')
|
||||
self.checkJoin(RFC2396_BASE, '', RFC2396_BASE)
|
||||
self.checkJoin(RFC2396_BASE, '../../../g', 'http://a/../g')
|
||||
self.checkJoin(RFC2396_BASE, '../../../../g', 'http://a/../../g')
|
||||
self.checkJoin(RFC2396_BASE, '/./g', 'http://a/./g')
|
||||
self.checkJoin(RFC2396_BASE, '/../g', 'http://a/../g')
|
||||
self.checkJoin(RFC2396_BASE, 'g.', 'http://a/b/c/g.')
|
||||
self.checkJoin(RFC2396_BASE, '.g', 'http://a/b/c/.g')
|
||||
self.checkJoin(RFC2396_BASE, 'g..', 'http://a/b/c/g..')
|
||||
self.checkJoin(RFC2396_BASE, '..g', 'http://a/b/c/..g')
|
||||
self.checkJoin(RFC2396_BASE, './../g', 'http://a/b/g')
|
||||
self.checkJoin(RFC2396_BASE, './g/.', 'http://a/b/c/g/')
|
||||
self.checkJoin(RFC2396_BASE, 'g/./h', 'http://a/b/c/g/h')
|
||||
self.checkJoin(RFC2396_BASE, 'g/../h', 'http://a/b/c/h')
|
||||
self.checkJoin(RFC2396_BASE, 'g;x=1/./y', 'http://a/b/c/g;x=1/y')
|
||||
self.checkJoin(RFC2396_BASE, 'g;x=1/../y', 'http://a/b/c/y')
|
||||
self.checkJoin(RFC2396_BASE, 'g?y/./x', 'http://a/b/c/g?y/./x')
|
||||
self.checkJoin(RFC2396_BASE, 'g?y/../x', 'http://a/b/c/g?y/../x')
|
||||
self.checkJoin(RFC2396_BASE, 'g#s/./x', 'http://a/b/c/g#s/./x')
|
||||
self.checkJoin(RFC2396_BASE, 'g#s/../x', 'http://a/b/c/g#s/../x')
|
||||
|
||||
def test_RFC3986(self):
|
||||
# Test cases from RFC3986
|
||||
self.checkJoin(RFC3986_BASE, '?y','http://a/b/c/d;p?y')
|
||||
self.checkJoin(RFC2396_BASE, ';x', 'http://a/b/c/;x')
|
||||
self.checkJoin(RFC3986_BASE, 'g:h','g:h')
|
||||
self.checkJoin(RFC3986_BASE, 'g','http://a/b/c/g')
|
||||
self.checkJoin(RFC3986_BASE, './g','http://a/b/c/g')
|
||||
self.checkJoin(RFC3986_BASE, 'g/','http://a/b/c/g/')
|
||||
self.checkJoin(RFC3986_BASE, '/g','http://a/g')
|
||||
self.checkJoin(RFC3986_BASE, '//g','http://g')
|
||||
self.checkJoin(RFC3986_BASE, '?y','http://a/b/c/d;p?y')
|
||||
self.checkJoin(RFC3986_BASE, 'g?y','http://a/b/c/g?y')
|
||||
self.checkJoin(RFC3986_BASE, '#s','http://a/b/c/d;p?q#s')
|
||||
self.checkJoin(RFC3986_BASE, 'g#s','http://a/b/c/g#s')
|
||||
self.checkJoin(RFC3986_BASE, 'g?y#s','http://a/b/c/g?y#s')
|
||||
self.checkJoin(RFC3986_BASE, ';x','http://a/b/c/;x')
|
||||
self.checkJoin(RFC3986_BASE, 'g;x','http://a/b/c/g;x')
|
||||
self.checkJoin(RFC3986_BASE, 'g;x?y#s','http://a/b/c/g;x?y#s')
|
||||
self.checkJoin(RFC3986_BASE, '','http://a/b/c/d;p?q')
|
||||
self.checkJoin(RFC3986_BASE, '.','http://a/b/c/')
|
||||
self.checkJoin(RFC3986_BASE, './','http://a/b/c/')
|
||||
self.checkJoin(RFC3986_BASE, '..','http://a/b/')
|
||||
self.checkJoin(RFC3986_BASE, '../','http://a/b/')
|
||||
self.checkJoin(RFC3986_BASE, '../g','http://a/b/g')
|
||||
self.checkJoin(RFC3986_BASE, '../..','http://a/')
|
||||
self.checkJoin(RFC3986_BASE, '../../','http://a/')
|
||||
self.checkJoin(RFC3986_BASE, '../../g','http://a/g')
|
||||
|
||||
#Abnormal Examples
|
||||
|
||||
# The 'abnormal scenarios' are incompatible with RFC2986 parsing
|
||||
# Tests are here for reference.
|
||||
|
||||
#self.checkJoin(RFC3986_BASE, '../../../g','http://a/g')
|
||||
#self.checkJoin(RFC3986_BASE, '../../../../g','http://a/g')
|
||||
#self.checkJoin(RFC3986_BASE, '/./g','http://a/g')
|
||||
#self.checkJoin(RFC3986_BASE, '/../g','http://a/g')
|
||||
|
||||
self.checkJoin(RFC3986_BASE, 'g.','http://a/b/c/g.')
|
||||
self.checkJoin(RFC3986_BASE, '.g','http://a/b/c/.g')
|
||||
self.checkJoin(RFC3986_BASE, 'g..','http://a/b/c/g..')
|
||||
self.checkJoin(RFC3986_BASE, '..g','http://a/b/c/..g')
|
||||
self.checkJoin(RFC3986_BASE, './../g','http://a/b/g')
|
||||
self.checkJoin(RFC3986_BASE, './g/.','http://a/b/c/g/')
|
||||
self.checkJoin(RFC3986_BASE, 'g/./h','http://a/b/c/g/h')
|
||||
self.checkJoin(RFC3986_BASE, 'g/../h','http://a/b/c/h')
|
||||
self.checkJoin(RFC3986_BASE, 'g;x=1/./y','http://a/b/c/g;x=1/y')
|
||||
self.checkJoin(RFC3986_BASE, 'g;x=1/../y','http://a/b/c/y')
|
||||
self.checkJoin(RFC3986_BASE, 'g?y/./x','http://a/b/c/g?y/./x')
|
||||
self.checkJoin(RFC3986_BASE, 'g?y/../x','http://a/b/c/g?y/../x')
|
||||
self.checkJoin(RFC3986_BASE, 'g#s/./x','http://a/b/c/g#s/./x')
|
||||
self.checkJoin(RFC3986_BASE, 'g#s/../x','http://a/b/c/g#s/../x')
|
||||
#self.checkJoin(RFC3986_BASE, 'http:g','http:g') # strict parser
|
||||
self.checkJoin(RFC3986_BASE, 'http:g','http://a/b/c/g') #relaxed parser
|
||||
|
||||
# Test for issue9721
|
||||
self.checkJoin('http://a/b/c/de', ';x','http://a/b/c/;x')
|
||||
|
||||
def test_urljoins(self):
|
||||
self.checkJoin(SIMPLE_BASE, 'g:h','g:h')
|
||||
self.checkJoin(SIMPLE_BASE, 'http:g','http://a/b/c/g')
|
||||
self.checkJoin(SIMPLE_BASE, 'http:','http://a/b/c/d')
|
||||
self.checkJoin(SIMPLE_BASE, 'g','http://a/b/c/g')
|
||||
self.checkJoin(SIMPLE_BASE, './g','http://a/b/c/g')
|
||||
self.checkJoin(SIMPLE_BASE, 'g/','http://a/b/c/g/')
|
||||
self.checkJoin(SIMPLE_BASE, '/g','http://a/g')
|
||||
self.checkJoin(SIMPLE_BASE, '//g','http://g')
|
||||
self.checkJoin(SIMPLE_BASE, '?y','http://a/b/c/d?y')
|
||||
self.checkJoin(SIMPLE_BASE, 'g?y','http://a/b/c/g?y')
|
||||
self.checkJoin(SIMPLE_BASE, 'g?y/./x','http://a/b/c/g?y/./x')
|
||||
self.checkJoin(SIMPLE_BASE, '.','http://a/b/c/')
|
||||
self.checkJoin(SIMPLE_BASE, './','http://a/b/c/')
|
||||
self.checkJoin(SIMPLE_BASE, '..','http://a/b/')
|
||||
self.checkJoin(SIMPLE_BASE, '../','http://a/b/')
|
||||
self.checkJoin(SIMPLE_BASE, '../g','http://a/b/g')
|
||||
self.checkJoin(SIMPLE_BASE, '../..','http://a/')
|
||||
self.checkJoin(SIMPLE_BASE, '../../g','http://a/g')
|
||||
self.checkJoin(SIMPLE_BASE, '../../../g','http://a/../g')
|
||||
self.checkJoin(SIMPLE_BASE, './../g','http://a/b/g')
|
||||
self.checkJoin(SIMPLE_BASE, './g/.','http://a/b/c/g/')
|
||||
self.checkJoin(SIMPLE_BASE, '/./g','http://a/./g')
|
||||
self.checkJoin(SIMPLE_BASE, 'g/./h','http://a/b/c/g/h')
|
||||
self.checkJoin(SIMPLE_BASE, 'g/../h','http://a/b/c/h')
|
||||
self.checkJoin(SIMPLE_BASE, 'http:g','http://a/b/c/g')
|
||||
self.checkJoin(SIMPLE_BASE, 'http:','http://a/b/c/d')
|
||||
self.checkJoin(SIMPLE_BASE, 'http:?y','http://a/b/c/d?y')
|
||||
self.checkJoin(SIMPLE_BASE, 'http:g?y','http://a/b/c/g?y')
|
||||
self.checkJoin(SIMPLE_BASE, 'http:g?y/./x','http://a/b/c/g?y/./x')
|
||||
self.checkJoin('http:///', '..','http:///')
|
||||
self.checkJoin('', 'http://a/b/c/g?y/./x','http://a/b/c/g?y/./x')
|
||||
self.checkJoin('', 'http://a/./g', 'http://a/./g')
|
||||
self.checkJoin('svn://pathtorepo/dir1', 'dir2', 'svn://pathtorepo/dir2')
|
||||
self.checkJoin('svn+ssh://pathtorepo/dir1', 'dir2', 'svn+ssh://pathtorepo/dir2')
|
||||
|
||||
def test_RFC2732(self):
|
||||
str_cases = [
|
||||
('http://Test.python.org:5432/foo/', 'test.python.org', 5432),
|
||||
('http://12.34.56.78:5432/foo/', '12.34.56.78', 5432),
|
||||
('http://[::1]:5432/foo/', '::1', 5432),
|
||||
('http://[dead:beef::1]:5432/foo/', 'dead:beef::1', 5432),
|
||||
('http://[dead:beef::]:5432/foo/', 'dead:beef::', 5432),
|
||||
('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]:5432/foo/',
|
||||
'dead:beef:cafe:5417:affe:8fa3:deaf:feed', 5432),
|
||||
('http://[::12.34.56.78]:5432/foo/', '::12.34.56.78', 5432),
|
||||
('http://[::ffff:12.34.56.78]:5432/foo/',
|
||||
'::ffff:12.34.56.78', 5432),
|
||||
('http://Test.python.org/foo/', 'test.python.org', None),
|
||||
('http://12.34.56.78/foo/', '12.34.56.78', None),
|
||||
('http://[::1]/foo/', '::1', None),
|
||||
('http://[dead:beef::1]/foo/', 'dead:beef::1', None),
|
||||
('http://[dead:beef::]/foo/', 'dead:beef::', None),
|
||||
('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]/foo/',
|
||||
'dead:beef:cafe:5417:affe:8fa3:deaf:feed', None),
|
||||
('http://[::12.34.56.78]/foo/', '::12.34.56.78', None),
|
||||
('http://[::ffff:12.34.56.78]/foo/',
|
||||
'::ffff:12.34.56.78', None),
|
||||
]
|
||||
def _encode(t):
|
||||
return t[0].encode('ascii'), t[1].encode('ascii'), t[2]
|
||||
bytes_cases = [_encode(x) for x in str_cases]
|
||||
for url, hostname, port in str_cases + bytes_cases:
|
||||
urlparsed = urllib.parse.urlparse(url)
|
||||
self.assertEqual((urlparsed.hostname, urlparsed.port) , (hostname, port))
|
||||
|
||||
str_cases = [
|
||||
'http://::12.34.56.78]/',
|
||||
'http://[::1/foo/',
|
||||
'ftp://[::1/foo/bad]/bad',
|
||||
'http://[::1/foo/bad]/bad',
|
||||
'http://[::ffff:12.34.56.78']
|
||||
bytes_cases = [x.encode('ascii') for x in str_cases]
|
||||
for invalid_url in str_cases + bytes_cases:
|
||||
self.assertRaises(ValueError, urllib.parse.urlparse, invalid_url)
|
||||
|
||||
def test_urldefrag(self):
|
||||
str_cases = [
|
||||
('http://python.org#frag', 'http://python.org', 'frag'),
|
||||
('http://python.org', 'http://python.org', ''),
|
||||
('http://python.org/#frag', 'http://python.org/', 'frag'),
|
||||
('http://python.org/', 'http://python.org/', ''),
|
||||
('http://python.org/?q#frag', 'http://python.org/?q', 'frag'),
|
||||
('http://python.org/?q', 'http://python.org/?q', ''),
|
||||
('http://python.org/p#frag', 'http://python.org/p', 'frag'),
|
||||
('http://python.org/p?q', 'http://python.org/p?q', ''),
|
||||
(RFC1808_BASE, 'http://a/b/c/d;p?q', 'f'),
|
||||
(RFC2396_BASE, 'http://a/b/c/d;p?q', ''),
|
||||
]
|
||||
def _encode(t):
|
||||
return type(t)(x.encode('ascii') for x in t)
|
||||
bytes_cases = [_encode(x) for x in str_cases]
|
||||
for url, defrag, frag in str_cases + bytes_cases:
|
||||
result = urllib.parse.urldefrag(url)
|
||||
self.assertEqual(result.geturl(), url)
|
||||
self.assertEqual(result, (defrag, frag))
|
||||
self.assertEqual(result.url, defrag)
|
||||
self.assertEqual(result.fragment, frag)
|
||||
|
||||
def test_urlsplit_attributes(self):
|
||||
url = "HTTP://WWW.PYTHON.ORG/doc/#frag"
|
||||
p = urllib.parse.urlsplit(url)
|
||||
self.assertEqual(p.scheme, "http")
|
||||
self.assertEqual(p.netloc, "WWW.PYTHON.ORG")
|
||||
self.assertEqual(p.path, "/doc/")
|
||||
self.assertEqual(p.query, "")
|
||||
self.assertEqual(p.fragment, "frag")
|
||||
self.assertEqual(p.username, None)
|
||||
self.assertEqual(p.password, None)
|
||||
self.assertEqual(p.hostname, "www.python.org")
|
||||
self.assertEqual(p.port, None)
|
||||
# geturl() won't return exactly the original URL in this case
|
||||
# since the scheme is always case-normalized
|
||||
# We handle this by ignoring the first 4 characters of the URL
|
||||
self.assertEqual(p.geturl()[4:], url[4:])
|
||||
|
||||
url = "http://User:Pass@www.python.org:080/doc/?query=yes#frag"
|
||||
p = urllib.parse.urlsplit(url)
|
||||
self.assertEqual(p.scheme, "http")
|
||||
self.assertEqual(p.netloc, "User:Pass@www.python.org:080")
|
||||
self.assertEqual(p.path, "/doc/")
|
||||
self.assertEqual(p.query, "query=yes")
|
||||
self.assertEqual(p.fragment, "frag")
|
||||
self.assertEqual(p.username, "User")
|
||||
self.assertEqual(p.password, "Pass")
|
||||
self.assertEqual(p.hostname, "www.python.org")
|
||||
self.assertEqual(p.port, 80)
|
||||
self.assertEqual(p.geturl(), url)
|
||||
|
||||
# Addressing issue1698, which suggests Username can contain
|
||||
# "@" characters. Though not RFC compliant, many ftp sites allow
|
||||
# and request email addresses as usernames.
|
||||
|
||||
url = "http://User@example.com:Pass@www.python.org:080/doc/?query=yes#frag"
|
||||
p = urllib.parse.urlsplit(url)
|
||||
self.assertEqual(p.scheme, "http")
|
||||
self.assertEqual(p.netloc, "User@example.com:Pass@www.python.org:080")
|
||||
self.assertEqual(p.path, "/doc/")
|
||||
self.assertEqual(p.query, "query=yes")
|
||||
self.assertEqual(p.fragment, "frag")
|
||||
self.assertEqual(p.username, "User@example.com")
|
||||
self.assertEqual(p.password, "Pass")
|
||||
self.assertEqual(p.hostname, "www.python.org")
|
||||
self.assertEqual(p.port, 80)
|
||||
self.assertEqual(p.geturl(), url)
|
||||
|
||||
# And check them all again, only with bytes this time
|
||||
url = b"HTTP://WWW.PYTHON.ORG/doc/#frag"
|
||||
p = urllib.parse.urlsplit(url)
|
||||
self.assertEqual(p.scheme, b"http")
|
||||
self.assertEqual(p.netloc, b"WWW.PYTHON.ORG")
|
||||
self.assertEqual(p.path, b"/doc/")
|
||||
self.assertEqual(p.query, b"")
|
||||
self.assertEqual(p.fragment, b"frag")
|
||||
self.assertEqual(p.username, None)
|
||||
self.assertEqual(p.password, None)
|
||||
self.assertEqual(p.hostname, b"www.python.org")
|
||||
self.assertEqual(p.port, None)
|
||||
self.assertEqual(p.geturl()[4:], url[4:])
|
||||
|
||||
url = b"http://User:Pass@www.python.org:080/doc/?query=yes#frag"
|
||||
p = urllib.parse.urlsplit(url)
|
||||
self.assertEqual(p.scheme, b"http")
|
||||
self.assertEqual(p.netloc, b"User:Pass@www.python.org:080")
|
||||
self.assertEqual(p.path, b"/doc/")
|
||||
self.assertEqual(p.query, b"query=yes")
|
||||
self.assertEqual(p.fragment, b"frag")
|
||||
self.assertEqual(p.username, b"User")
|
||||
self.assertEqual(p.password, b"Pass")
|
||||
self.assertEqual(p.hostname, b"www.python.org")
|
||||
self.assertEqual(p.port, 80)
|
||||
self.assertEqual(p.geturl(), url)
|
||||
|
||||
url = b"http://User@example.com:Pass@www.python.org:080/doc/?query=yes#frag"
|
||||
p = urllib.parse.urlsplit(url)
|
||||
self.assertEqual(p.scheme, b"http")
|
||||
self.assertEqual(p.netloc, b"User@example.com:Pass@www.python.org:080")
|
||||
self.assertEqual(p.path, b"/doc/")
|
||||
self.assertEqual(p.query, b"query=yes")
|
||||
self.assertEqual(p.fragment, b"frag")
|
||||
self.assertEqual(p.username, b"User@example.com")
|
||||
self.assertEqual(p.password, b"Pass")
|
||||
self.assertEqual(p.hostname, b"www.python.org")
|
||||
self.assertEqual(p.port, 80)
|
||||
self.assertEqual(p.geturl(), url)
|
||||
|
||||
# Verify an illegal port is returned as None
|
||||
url = b"HTTP://WWW.PYTHON.ORG:65536/doc/#frag"
|
||||
p = urllib.parse.urlsplit(url)
|
||||
self.assertEqual(p.port, None)
|
||||
|
||||
def test_attributes_bad_port(self):
|
||||
"""Check handling of non-integer ports."""
|
||||
p = urllib.parse.urlsplit("http://www.example.net:foo")
|
||||
self.assertEqual(p.netloc, "www.example.net:foo")
|
||||
self.assertRaises(ValueError, lambda: p.port)
|
||||
|
||||
p = urllib.parse.urlparse("http://www.example.net:foo")
|
||||
self.assertEqual(p.netloc, "www.example.net:foo")
|
||||
self.assertRaises(ValueError, lambda: p.port)
|
||||
|
||||
# Once again, repeat ourselves to test bytes
|
||||
p = urllib.parse.urlsplit(b"http://www.example.net:foo")
|
||||
self.assertEqual(p.netloc, b"www.example.net:foo")
|
||||
self.assertRaises(ValueError, lambda: p.port)
|
||||
|
||||
p = urllib.parse.urlparse(b"http://www.example.net:foo")
|
||||
self.assertEqual(p.netloc, b"www.example.net:foo")
|
||||
self.assertRaises(ValueError, lambda: p.port)
|
||||
|
||||
def test_attributes_without_netloc(self):
|
||||
# This example is straight from RFC 3261. It looks like it
|
||||
# should allow the username, hostname, and port to be filled
|
||||
# in, but doesn't. Since it's a URI and doesn't use the
|
||||
# scheme://netloc syntax, the netloc and related attributes
|
||||
# should be left empty.
|
||||
uri = "sip:alice@atlanta.com;maddr=239.255.255.1;ttl=15"
|
||||
p = urllib.parse.urlsplit(uri)
|
||||
self.assertEqual(p.netloc, "")
|
||||
self.assertEqual(p.username, None)
|
||||
self.assertEqual(p.password, None)
|
||||
self.assertEqual(p.hostname, None)
|
||||
self.assertEqual(p.port, None)
|
||||
self.assertEqual(p.geturl(), uri)
|
||||
|
||||
p = urllib.parse.urlparse(uri)
|
||||
self.assertEqual(p.netloc, "")
|
||||
self.assertEqual(p.username, None)
|
||||
self.assertEqual(p.password, None)
|
||||
self.assertEqual(p.hostname, None)
|
||||
self.assertEqual(p.port, None)
|
||||
self.assertEqual(p.geturl(), uri)
|
||||
|
||||
# You guessed it, repeating the test with bytes input
|
||||
uri = b"sip:alice@atlanta.com;maddr=239.255.255.1;ttl=15"
|
||||
p = urllib.parse.urlsplit(uri)
|
||||
self.assertEqual(p.netloc, b"")
|
||||
self.assertEqual(p.username, None)
|
||||
self.assertEqual(p.password, None)
|
||||
self.assertEqual(p.hostname, None)
|
||||
self.assertEqual(p.port, None)
|
||||
self.assertEqual(p.geturl(), uri)
|
||||
|
||||
p = urllib.parse.urlparse(uri)
|
||||
self.assertEqual(p.netloc, b"")
|
||||
self.assertEqual(p.username, None)
|
||||
self.assertEqual(p.password, None)
|
||||
self.assertEqual(p.hostname, None)
|
||||
self.assertEqual(p.port, None)
|
||||
self.assertEqual(p.geturl(), uri)
|
||||
|
||||
def test_noslash(self):
|
||||
# Issue 1637: http://foo.com?query is legal
|
||||
self.assertEqual(urllib.parse.urlparse("http://example.com?blahblah=/foo"),
|
||||
('http', 'example.com', '', '', 'blahblah=/foo', ''))
|
||||
self.assertEqual(urllib.parse.urlparse(b"http://example.com?blahblah=/foo"),
|
||||
(b'http', b'example.com', b'', b'', b'blahblah=/foo', b''))
|
||||
|
||||
def test_withoutscheme(self):
|
||||
# Test urlparse without scheme
|
||||
# Issue 754016: urlparse goes wrong with IP:port without scheme
|
||||
# RFC 1808 specifies that netloc should start with //, urlparse expects
|
||||
# the same, otherwise it classifies the portion of url as path.
|
||||
self.assertEqual(urllib.parse.urlparse("path"),
|
||||
('','','path','','',''))
|
||||
self.assertEqual(urllib.parse.urlparse("//www.python.org:80"),
|
||||
('','www.python.org:80','','','',''))
|
||||
self.assertEqual(urllib.parse.urlparse("http://www.python.org:80"),
|
||||
('http','www.python.org:80','','','',''))
|
||||
# Repeat for bytes input
|
||||
self.assertEqual(urllib.parse.urlparse(b"path"),
|
||||
(b'',b'',b'path',b'',b'',b''))
|
||||
self.assertEqual(urllib.parse.urlparse(b"//www.python.org:80"),
|
||||
(b'',b'www.python.org:80',b'',b'',b'',b''))
|
||||
self.assertEqual(urllib.parse.urlparse(b"http://www.python.org:80"),
|
||||
(b'http',b'www.python.org:80',b'',b'',b'',b''))
|
||||
|
||||
def test_portseparator(self):
|
||||
# Issue 754016 makes changes for port separator ':' from scheme separator
|
||||
self.assertEqual(urllib.parse.urlparse("path:80"),
|
||||
('','','path:80','','',''))
|
||||
self.assertEqual(urllib.parse.urlparse("http:"),('http','','','','',''))
|
||||
self.assertEqual(urllib.parse.urlparse("https:"),('https','','','','',''))
|
||||
self.assertEqual(urllib.parse.urlparse("http://www.python.org:80"),
|
||||
('http','www.python.org:80','','','',''))
|
||||
# As usual, need to check bytes input as well
|
||||
self.assertEqual(urllib.parse.urlparse(b"path:80"),
|
||||
(b'',b'',b'path:80',b'',b'',b''))
|
||||
self.assertEqual(urllib.parse.urlparse(b"http:"),(b'http',b'',b'',b'',b'',b''))
|
||||
self.assertEqual(urllib.parse.urlparse(b"https:"),(b'https',b'',b'',b'',b'',b''))
|
||||
self.assertEqual(urllib.parse.urlparse(b"http://www.python.org:80"),
|
||||
(b'http',b'www.python.org:80',b'',b'',b'',b''))
|
||||
|
||||
def test_usingsys(self):
|
||||
# Issue 3314: sys module is used in the error
|
||||
self.assertRaises(TypeError, urllib.parse.urlencode, "foo")
|
||||
|
||||
def test_anyscheme(self):
|
||||
# Issue 7904: s3://foo.com/stuff has netloc "foo.com".
|
||||
self.assertEqual(urllib.parse.urlparse("s3://foo.com/stuff"),
|
||||
('s3', 'foo.com', '/stuff', '', '', ''))
|
||||
self.assertEqual(urllib.parse.urlparse("x-newscheme://foo.com/stuff"),
|
||||
('x-newscheme', 'foo.com', '/stuff', '', '', ''))
|
||||
self.assertEqual(urllib.parse.urlparse("x-newscheme://foo.com/stuff?query#fragment"),
|
||||
('x-newscheme', 'foo.com', '/stuff', '', 'query', 'fragment'))
|
||||
self.assertEqual(urllib.parse.urlparse("x-newscheme://foo.com/stuff?query"),
|
||||
('x-newscheme', 'foo.com', '/stuff', '', 'query', ''))
|
||||
|
||||
# And for bytes...
|
||||
self.assertEqual(urllib.parse.urlparse(b"s3://foo.com/stuff"),
|
||||
(b's3', b'foo.com', b'/stuff', b'', b'', b''))
|
||||
self.assertEqual(urllib.parse.urlparse(b"x-newscheme://foo.com/stuff"),
|
||||
(b'x-newscheme', b'foo.com', b'/stuff', b'', b'', b''))
|
||||
self.assertEqual(urllib.parse.urlparse(b"x-newscheme://foo.com/stuff?query#fragment"),
|
||||
(b'x-newscheme', b'foo.com', b'/stuff', b'', b'query', b'fragment'))
|
||||
self.assertEqual(urllib.parse.urlparse(b"x-newscheme://foo.com/stuff?query"),
|
||||
(b'x-newscheme', b'foo.com', b'/stuff', b'', b'query', b''))
|
||||
|
||||
def test_mixed_types_rejected(self):
|
||||
# Several functions that process either strings or ASCII encoded bytes
|
||||
# accept multiple arguments. Check they reject mixed type input
|
||||
with self.assertRaisesRegex(TypeError, "Cannot mix str"):
|
||||
urllib.parse.urlparse("www.python.org", b"http")
|
||||
with self.assertRaisesRegex(TypeError, "Cannot mix str"):
|
||||
urllib.parse.urlparse(b"www.python.org", "http")
|
||||
with self.assertRaisesRegex(TypeError, "Cannot mix str"):
|
||||
urllib.parse.urlsplit("www.python.org", b"http")
|
||||
with self.assertRaisesRegex(TypeError, "Cannot mix str"):
|
||||
urllib.parse.urlsplit(b"www.python.org", "http")
|
||||
with self.assertRaisesRegex(TypeError, "Cannot mix str"):
|
||||
urllib.parse.urlunparse(( b"http", "www.python.org","","","",""))
|
||||
with self.assertRaisesRegex(TypeError, "Cannot mix str"):
|
||||
urllib.parse.urlunparse(("http", b"www.python.org","","","",""))
|
||||
with self.assertRaisesRegex(TypeError, "Cannot mix str"):
|
||||
urllib.parse.urlunsplit((b"http", "www.python.org","","",""))
|
||||
with self.assertRaisesRegex(TypeError, "Cannot mix str"):
|
||||
urllib.parse.urlunsplit(("http", b"www.python.org","","",""))
|
||||
with self.assertRaisesRegex(TypeError, "Cannot mix str"):
|
||||
urllib.parse.urljoin("http://python.org", b"http://python.org")
|
||||
with self.assertRaisesRegex(TypeError, "Cannot mix str"):
|
||||
urllib.parse.urljoin(b"http://python.org", "http://python.org")
|
||||
|
||||
def _check_result_type(self, str_type):
|
||||
num_args = len(str_type._fields)
|
||||
bytes_type = str_type._encoded_counterpart
|
||||
self.assertIs(bytes_type._decoded_counterpart, str_type)
|
||||
str_args = ('',) * num_args
|
||||
bytes_args = (b'',) * num_args
|
||||
str_result = str_type(*str_args)
|
||||
bytes_result = bytes_type(*bytes_args)
|
||||
encoding = 'ascii'
|
||||
errors = 'strict'
|
||||
self.assertEqual(str_result, str_args)
|
||||
self.assertEqual(bytes_result.decode(), str_args)
|
||||
self.assertEqual(bytes_result.decode(), str_result)
|
||||
self.assertEqual(bytes_result.decode(encoding), str_args)
|
||||
self.assertEqual(bytes_result.decode(encoding), str_result)
|
||||
self.assertEqual(bytes_result.decode(encoding, errors), str_args)
|
||||
self.assertEqual(bytes_result.decode(encoding, errors), str_result)
|
||||
self.assertEqual(bytes_result, bytes_args)
|
||||
self.assertEqual(str_result.encode(), bytes_args)
|
||||
self.assertEqual(str_result.encode(), bytes_result)
|
||||
self.assertEqual(str_result.encode(encoding), bytes_args)
|
||||
self.assertEqual(str_result.encode(encoding), bytes_result)
|
||||
self.assertEqual(str_result.encode(encoding, errors), bytes_args)
|
||||
self.assertEqual(str_result.encode(encoding, errors), bytes_result)
|
||||
|
||||
def test_result_pairs(self):
|
||||
# Check encoding and decoding between result pairs
|
||||
result_types = [
|
||||
urllib.parse.DefragResult,
|
||||
urllib.parse.SplitResult,
|
||||
urllib.parse.ParseResult,
|
||||
]
|
||||
for result_type in result_types:
|
||||
self._check_result_type(result_type)
|
||||
|
||||
def test_parse_qs_encoding(self):
|
||||
result = urllib.parse.parse_qs("key=\u0141%E9", encoding="latin-1")
|
||||
self.assertEqual(result, {'key': ['\u0141\xE9']})
|
||||
result = urllib.parse.parse_qs("key=\u0141%C3%A9", encoding="utf-8")
|
||||
self.assertEqual(result, {'key': ['\u0141\xE9']})
|
||||
result = urllib.parse.parse_qs("key=\u0141%C3%A9", encoding="ascii")
|
||||
self.assertEqual(result, {'key': ['\u0141\ufffd\ufffd']})
|
||||
result = urllib.parse.parse_qs("key=\u0141%E9-", encoding="ascii")
|
||||
self.assertEqual(result, {'key': ['\u0141\ufffd-']})
|
||||
result = urllib.parse.parse_qs("key=\u0141%E9-", encoding="ascii",
|
||||
errors="ignore")
|
||||
self.assertEqual(result, {'key': ['\u0141-']})
|
||||
|
||||
def test_parse_qsl_encoding(self):
|
||||
result = urllib.parse.parse_qsl("key=\u0141%E9", encoding="latin-1")
|
||||
self.assertEqual(result, [('key', '\u0141\xE9')])
|
||||
result = urllib.parse.parse_qsl("key=\u0141%C3%A9", encoding="utf-8")
|
||||
self.assertEqual(result, [('key', '\u0141\xE9')])
|
||||
result = urllib.parse.parse_qsl("key=\u0141%C3%A9", encoding="ascii")
|
||||
self.assertEqual(result, [('key', '\u0141\ufffd\ufffd')])
|
||||
result = urllib.parse.parse_qsl("key=\u0141%E9-", encoding="ascii")
|
||||
self.assertEqual(result, [('key', '\u0141\ufffd-')])
|
||||
result = urllib.parse.parse_qsl("key=\u0141%E9-", encoding="ascii",
|
||||
errors="ignore")
|
||||
self.assertEqual(result, [('key', '\u0141-')])
|
||||
|
||||
def test_splitnport(self):
|
||||
# Normal cases are exercised by other tests; ensure that we also
|
||||
# catch cases with no port specified. (testcase ensuring coverage)
|
||||
result = urllib.parse.splitnport('parrot:88')
|
||||
self.assertEqual(result, ('parrot', 88))
|
||||
result = urllib.parse.splitnport('parrot')
|
||||
self.assertEqual(result, ('parrot', -1))
|
||||
result = urllib.parse.splitnport('parrot', 55)
|
||||
self.assertEqual(result, ('parrot', 55))
|
||||
result = urllib.parse.splitnport('parrot:')
|
||||
self.assertEqual(result, ('parrot', None))
|
||||
|
||||
def test_splitquery(self):
|
||||
# Normal cases are exercised by other tests; ensure that we also
|
||||
# catch cases with no port specified (testcase ensuring coverage)
|
||||
result = urllib.parse.splitquery('http://python.org/fake?foo=bar')
|
||||
self.assertEqual(result, ('http://python.org/fake', 'foo=bar'))
|
||||
result = urllib.parse.splitquery('http://python.org/fake?foo=bar?')
|
||||
self.assertEqual(result, ('http://python.org/fake?foo=bar', ''))
|
||||
result = urllib.parse.splitquery('http://python.org/fake')
|
||||
self.assertEqual(result, ('http://python.org/fake', None))
|
||||
|
||||
def test_splitvalue(self):
|
||||
# Normal cases are exercised by other tests; test pathological cases
|
||||
# with no key/value pairs. (testcase ensuring coverage)
|
||||
result = urllib.parse.splitvalue('foo=bar')
|
||||
self.assertEqual(result, ('foo', 'bar'))
|
||||
result = urllib.parse.splitvalue('foo=')
|
||||
self.assertEqual(result, ('foo', ''))
|
||||
result = urllib.parse.splitvalue('foobar')
|
||||
self.assertEqual(result, ('foobar', None))
|
||||
|
||||
def test_to_bytes(self):
|
||||
result = urllib.parse.to_bytes('http://www.python.org')
|
||||
self.assertEqual(result, 'http://www.python.org')
|
||||
self.assertRaises(UnicodeError, urllib.parse.to_bytes,
|
||||
'http://www.python.org/medi\u00e6val')
|
||||
|
||||
def test_urlencode_sequences(self):
|
||||
# Other tests incidentally urlencode things; test non-covered cases:
|
||||
# Sequence and object values.
|
||||
result = urllib.parse.urlencode({'a': [1, 2], 'b': (3, 4, 5)}, True)
|
||||
# we cannot rely on ordering here
|
||||
assert set(result.split('&')) == {'a=1', 'a=2', 'b=3', 'b=4', 'b=5'}
|
||||
|
||||
class Trivial:
|
||||
def __str__(self):
|
||||
return 'trivial'
|
||||
|
||||
result = urllib.parse.urlencode({'a': Trivial()}, True)
|
||||
self.assertEqual(result, 'a=trivial')
|
||||
|
||||
def test_quote_from_bytes(self):
|
||||
self.assertRaises(TypeError, urllib.parse.quote_from_bytes, 'foo')
|
||||
result = urllib.parse.quote_from_bytes(b'archaeological arcana')
|
||||
self.assertEqual(result, 'archaeological%20arcana')
|
||||
result = urllib.parse.quote_from_bytes(b'')
|
||||
self.assertEqual(result, '')
|
||||
|
||||
def test_unquote_to_bytes(self):
|
||||
result = urllib.parse.unquote_to_bytes('abc%20def')
|
||||
self.assertEqual(result, b'abc def')
|
||||
result = urllib.parse.unquote_to_bytes('')
|
||||
self.assertEqual(result, b'')
|
||||
|
||||
def test_quote_errors(self):
|
||||
self.assertRaises(TypeError, urllib.parse.quote, b'foo',
|
||||
encoding='utf-8')
|
||||
self.assertRaises(TypeError, urllib.parse.quote, b'foo', errors='strict')
|
||||
|
||||
def test_issue14072(self):
|
||||
p1 = urllib.parse.urlsplit('tel:+31-641044153')
|
||||
self.assertEqual(p1.scheme, 'tel')
|
||||
self.assertEqual(p1.path, '+31-641044153')
|
||||
p2 = urllib.parse.urlsplit('tel:+31641044153')
|
||||
self.assertEqual(p2.scheme, 'tel')
|
||||
self.assertEqual(p2.path, '+31641044153')
|
||||
# assert the behavior for urlparse
|
||||
p1 = urllib.parse.urlparse('tel:+31-641044153')
|
||||
self.assertEqual(p1.scheme, 'tel')
|
||||
self.assertEqual(p1.path, '+31-641044153')
|
||||
p2 = urllib.parse.urlparse('tel:+31641044153')
|
||||
self.assertEqual(p2.scheme, 'tel')
|
||||
self.assertEqual(p2.path, '+31641044153')
|
||||
|
||||
def test_telurl_params(self):
|
||||
p1 = urllib.parse.urlparse('tel:123-4;phone-context=+1-650-516')
|
||||
self.assertEqual(p1.scheme, 'tel')
|
||||
self.assertEqual(p1.path, '123-4')
|
||||
self.assertEqual(p1.params, 'phone-context=+1-650-516')
|
||||
|
||||
p1 = urllib.parse.urlparse('tel:+1-201-555-0123')
|
||||
self.assertEqual(p1.scheme, 'tel')
|
||||
self.assertEqual(p1.path, '+1-201-555-0123')
|
||||
self.assertEqual(p1.params, '')
|
||||
|
||||
p1 = urllib.parse.urlparse('tel:7042;phone-context=example.com')
|
||||
self.assertEqual(p1.scheme, 'tel')
|
||||
self.assertEqual(p1.path, '7042')
|
||||
self.assertEqual(p1.params, 'phone-context=example.com')
|
||||
|
||||
p1 = urllib.parse.urlparse('tel:863-1234;phone-context=+1-914-555')
|
||||
self.assertEqual(p1.scheme, 'tel')
|
||||
self.assertEqual(p1.path, '863-1234')
|
||||
self.assertEqual(p1.params, 'phone-context=+1-914-555')
|
||||
|
||||
|
||||
def test_main():
|
||||
support.run_unittest(UrlParseTestCase)
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_main()
|
||||
|
|
@ -0,0 +1,974 @@
|
|||
"""Parse (absolute and relative) URLs.
|
||||
|
||||
urlparse module is based upon the following RFC specifications.
|
||||
|
||||
RFC 3986 (STD66): "Uniform Resource Identifiers" by T. Berners-Lee, R. Fielding
|
||||
and L. Masinter, January 2005.
|
||||
|
||||
RFC 2732 : "Format for Literal IPv6 Addresses in URL's by R.Hinden, B.Carpenter
|
||||
and L.Masinter, December 1999.
|
||||
|
||||
RFC 2396: "Uniform Resource Identifiers (URI)": Generic Syntax by T.
|
||||
Berners-Lee, R. Fielding, and L. Masinter, August 1998.
|
||||
|
||||
RFC 2368: "The mailto URL scheme", by P.Hoffman , L Masinter, J. Zawinski, July 1998.
|
||||
|
||||
RFC 1808: "Relative Uniform Resource Locators", by R. Fielding, UC Irvine, June
|
||||
1995.
|
||||
|
||||
RFC 1738: "Uniform Resource Locators (URL)" by T. Berners-Lee, L. Masinter, M.
|
||||
McCahill, December 1994
|
||||
|
||||
RFC 3986 is considered the current standard and any future changes to
|
||||
urlparse module should conform with it. The urlparse module is
|
||||
currently not entirely compliant with this RFC due to defacto
|
||||
scenarios for parsing, and for backward compatibility purposes, some
|
||||
parsing quirks from older RFCs are retained. The testcases in
|
||||
test_urlparse.py provides a good indicator of parsing behavior.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sys
|
||||
import collections
|
||||
|
||||
__all__ = ["urlparse", "urlunparse", "urljoin", "urldefrag",
|
||||
"urlsplit", "urlunsplit", "urlencode", "parse_qs",
|
||||
"parse_qsl", "quote", "quote_plus", "quote_from_bytes",
|
||||
"unquote", "unquote_plus", "unquote_to_bytes"]
|
||||
|
||||
# A classification of schemes ('' means apply by default)
|
||||
uses_relative = ['ftp', 'http', 'gopher', 'nntp', 'imap',
|
||||
'wais', 'file', 'https', 'shttp', 'mms',
|
||||
'prospero', 'rtsp', 'rtspu', '', 'sftp',
|
||||
'svn', 'svn+ssh']
|
||||
uses_netloc = ['ftp', 'http', 'gopher', 'nntp', 'telnet',
|
||||
'imap', 'wais', 'file', 'mms', 'https', 'shttp',
|
||||
'snews', 'prospero', 'rtsp', 'rtspu', 'rsync', '',
|
||||
'svn', 'svn+ssh', 'sftp', 'nfs', 'git', 'git+ssh']
|
||||
uses_params = ['ftp', 'hdl', 'prospero', 'http', 'imap',
|
||||
'https', 'shttp', 'rtsp', 'rtspu', 'sip', 'sips',
|
||||
'mms', '', 'sftp', 'tel']
|
||||
|
||||
# These are not actually used anymore, but should stay for backwards
|
||||
# compatibility. (They are undocumented, but have a public-looking name.)
|
||||
non_hierarchical = ['gopher', 'hdl', 'mailto', 'news',
|
||||
'telnet', 'wais', 'imap', 'snews', 'sip', 'sips']
|
||||
uses_query = ['http', 'wais', 'imap', 'https', 'shttp', 'mms',
|
||||
'gopher', 'rtsp', 'rtspu', 'sip', 'sips', '']
|
||||
uses_fragment = ['ftp', 'hdl', 'http', 'gopher', 'news',
|
||||
'nntp', 'wais', 'https', 'shttp', 'snews',
|
||||
'file', 'prospero', '']
|
||||
|
||||
# Characters valid in scheme names
|
||||
scheme_chars = ('abcdefghijklmnopqrstuvwxyz'
|
||||
'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
|
||||
'0123456789'
|
||||
'+-.')
|
||||
|
||||
# XXX: Consider replacing with functools.lru_cache
|
||||
MAX_CACHE_SIZE = 20
|
||||
_parse_cache = {}
|
||||
|
||||
def clear_cache():
|
||||
"""Clear the parse cache and the quoters cache."""
|
||||
_parse_cache.clear()
|
||||
_safe_quoters.clear()
|
||||
|
||||
|
||||
# Helpers for bytes handling
|
||||
# For 3.2, we deliberately require applications that
|
||||
# handle improperly quoted URLs to do their own
|
||||
# decoding and encoding. If valid use cases are
|
||||
# presented, we may relax this by using latin-1
|
||||
# decoding internally for 3.3
|
||||
_implicit_encoding = 'ascii'
|
||||
_implicit_errors = 'strict'
|
||||
|
||||
def _noop(obj):
|
||||
return obj
|
||||
|
||||
def _encode_result(obj, encoding=_implicit_encoding,
|
||||
errors=_implicit_errors):
|
||||
return obj.encode(encoding, errors)
|
||||
|
||||
def _decode_args(args, encoding=_implicit_encoding,
|
||||
errors=_implicit_errors):
|
||||
return tuple(x.decode(encoding, errors) if x else '' for x in args)
|
||||
|
||||
def _coerce_args(*args):
|
||||
# Invokes decode if necessary to create str args
|
||||
# and returns the coerced inputs along with
|
||||
# an appropriate result coercion function
|
||||
# - noop for str inputs
|
||||
# - encoding function otherwise
|
||||
str_input = isinstance(args[0], str)
|
||||
for arg in args[1:]:
|
||||
# We special-case the empty string to support the
|
||||
# "scheme=''" default argument to some functions
|
||||
if arg and isinstance(arg, str) != str_input:
|
||||
raise TypeError("Cannot mix str and non-str arguments")
|
||||
if str_input:
|
||||
return args + (_noop,)
|
||||
return _decode_args(args) + (_encode_result,)
|
||||
|
||||
# Result objects are more helpful than simple tuples
|
||||
class _ResultMixinStr(object):
|
||||
"""Standard approach to encoding parsed results from str to bytes"""
|
||||
__slots__ = ()
|
||||
|
||||
def encode(self, encoding='ascii', errors='strict'):
|
||||
return self._encoded_counterpart(*(x.encode(encoding, errors) for x in self))
|
||||
|
||||
|
||||
class _ResultMixinBytes(object):
|
||||
"""Standard approach to decoding parsed results from bytes to str"""
|
||||
__slots__ = ()
|
||||
|
||||
def decode(self, encoding='ascii', errors='strict'):
|
||||
return self._decoded_counterpart(*(x.decode(encoding, errors) for x in self))
|
||||
|
||||
|
||||
class _NetlocResultMixinBase(object):
|
||||
"""Shared methods for the parsed result objects containing a netloc element"""
|
||||
__slots__ = ()
|
||||
|
||||
@property
|
||||
def username(self):
|
||||
return self._userinfo[0]
|
||||
|
||||
@property
|
||||
def password(self):
|
||||
return self._userinfo[1]
|
||||
|
||||
@property
|
||||
def hostname(self):
|
||||
hostname = self._hostinfo[0]
|
||||
if not hostname:
|
||||
hostname = None
|
||||
elif hostname is not None:
|
||||
hostname = hostname.lower()
|
||||
return hostname
|
||||
|
||||
@property
|
||||
def port(self):
|
||||
port = self._hostinfo[1]
|
||||
if port is not None:
|
||||
port = int(port, 10)
|
||||
# Return None on an illegal port
|
||||
if not ( 0 <= port <= 65535):
|
||||
return None
|
||||
return port
|
||||
|
||||
|
||||
class _NetlocResultMixinStr(_NetlocResultMixinBase, _ResultMixinStr):
|
||||
__slots__ = ()
|
||||
|
||||
@property
|
||||
def _userinfo(self):
|
||||
netloc = self.netloc
|
||||
userinfo, have_info, hostinfo = netloc.rpartition('@')
|
||||
if have_info:
|
||||
username, have_password, password = userinfo.partition(':')
|
||||
if not have_password:
|
||||
password = None
|
||||
else:
|
||||
username = password = None
|
||||
return username, password
|
||||
|
||||
@property
|
||||
def _hostinfo(self):
|
||||
netloc = self.netloc
|
||||
_, _, hostinfo = netloc.rpartition('@')
|
||||
_, have_open_br, bracketed = hostinfo.partition('[')
|
||||
if have_open_br:
|
||||
hostname, _, port = bracketed.partition(']')
|
||||
_, have_port, port = port.partition(':')
|
||||
else:
|
||||
hostname, have_port, port = hostinfo.partition(':')
|
||||
if not have_port:
|
||||
port = None
|
||||
return hostname, port
|
||||
|
||||
|
||||
class _NetlocResultMixinBytes(_NetlocResultMixinBase, _ResultMixinBytes):
|
||||
__slots__ = ()
|
||||
|
||||
@property
|
||||
def _userinfo(self):
|
||||
netloc = self.netloc
|
||||
userinfo, have_info, hostinfo = netloc.rpartition(b'@')
|
||||
if have_info:
|
||||
username, have_password, password = userinfo.partition(b':')
|
||||
if not have_password:
|
||||
password = None
|
||||
else:
|
||||
username = password = None
|
||||
return username, password
|
||||
|
||||
@property
|
||||
def _hostinfo(self):
|
||||
netloc = self.netloc
|
||||
_, _, hostinfo = netloc.rpartition(b'@')
|
||||
_, have_open_br, bracketed = hostinfo.partition(b'[')
|
||||
if have_open_br:
|
||||
hostname, _, port = bracketed.partition(b']')
|
||||
_, have_port, port = port.partition(b':')
|
||||
else:
|
||||
hostname, have_port, port = hostinfo.partition(b':')
|
||||
if not have_port:
|
||||
port = None
|
||||
return hostname, port
|
||||
|
||||
|
||||
from collections import namedtuple
|
||||
|
||||
_DefragResultBase = namedtuple('DefragResult', 'url fragment')
|
||||
_SplitResultBase = namedtuple('SplitResult', 'scheme netloc path query fragment')
|
||||
_ParseResultBase = namedtuple('ParseResult', 'scheme netloc path params query fragment')
|
||||
|
||||
# For backwards compatibility, alias _NetlocResultMixinStr
|
||||
# ResultBase is no longer part of the documented API, but it is
|
||||
# retained since deprecating it isn't worth the hassle
|
||||
ResultBase = _NetlocResultMixinStr
|
||||
|
||||
# Structured result objects for string data
|
||||
class DefragResult(_DefragResultBase, _ResultMixinStr):
|
||||
__slots__ = ()
|
||||
def geturl(self):
|
||||
if self.fragment:
|
||||
return self.url + '#' + self.fragment
|
||||
else:
|
||||
return self.url
|
||||
|
||||
class SplitResult(_SplitResultBase, _NetlocResultMixinStr):
|
||||
__slots__ = ()
|
||||
def geturl(self):
|
||||
return urlunsplit(self)
|
||||
|
||||
class ParseResult(_ParseResultBase, _NetlocResultMixinStr):
|
||||
__slots__ = ()
|
||||
def geturl(self):
|
||||
return urlunparse(self)
|
||||
|
||||
# Structured result objects for bytes data
|
||||
class DefragResultBytes(_DefragResultBase, _ResultMixinBytes):
|
||||
__slots__ = ()
|
||||
def geturl(self):
|
||||
if self.fragment:
|
||||
return self.url + b'#' + self.fragment
|
||||
else:
|
||||
return self.url
|
||||
|
||||
class SplitResultBytes(_SplitResultBase, _NetlocResultMixinBytes):
|
||||
__slots__ = ()
|
||||
def geturl(self):
|
||||
return urlunsplit(self)
|
||||
|
||||
class ParseResultBytes(_ParseResultBase, _NetlocResultMixinBytes):
|
||||
__slots__ = ()
|
||||
def geturl(self):
|
||||
return urlunparse(self)
|
||||
|
||||
# Set up the encode/decode result pairs
|
||||
def _fix_result_transcoding():
|
||||
_result_pairs = (
|
||||
(DefragResult, DefragResultBytes),
|
||||
(SplitResult, SplitResultBytes),
|
||||
(ParseResult, ParseResultBytes),
|
||||
)
|
||||
for _decoded, _encoded in _result_pairs:
|
||||
_decoded._encoded_counterpart = _encoded
|
||||
_encoded._decoded_counterpart = _decoded
|
||||
|
||||
_fix_result_transcoding()
|
||||
del _fix_result_transcoding
|
||||
|
||||
def urlparse(url, scheme='', allow_fragments=True):
|
||||
"""Parse a URL into 6 components:
|
||||
<scheme>://<netloc>/<path>;<params>?<query>#<fragment>
|
||||
Return a 6-tuple: (scheme, netloc, path, params, query, fragment).
|
||||
Note that we don't break the components up in smaller bits
|
||||
(e.g. netloc is a single string) and we don't expand % escapes."""
|
||||
url, scheme, _coerce_result = _coerce_args(url, scheme)
|
||||
splitresult = urlsplit(url, scheme, allow_fragments)
|
||||
scheme, netloc, url, query, fragment = splitresult
|
||||
if scheme in uses_params and ';' in url:
|
||||
url, params = _splitparams(url)
|
||||
else:
|
||||
params = ''
|
||||
result = ParseResult(scheme, netloc, url, params, query, fragment)
|
||||
return _coerce_result(result)
|
||||
|
||||
def _splitparams(url):
|
||||
if '/' in url:
|
||||
i = url.find(';', url.rfind('/'))
|
||||
if i < 0:
|
||||
return url, ''
|
||||
else:
|
||||
i = url.find(';')
|
||||
return url[:i], url[i+1:]
|
||||
|
||||
def _splitnetloc(url, start=0):
|
||||
delim = len(url) # position of end of domain part of url, default is end
|
||||
for c in '/?#': # look for delimiters; the order is NOT important
|
||||
wdelim = url.find(c, start) # find first of this delim
|
||||
if wdelim >= 0: # if found
|
||||
delim = min(delim, wdelim) # use earliest delim position
|
||||
return url[start:delim], url[delim:] # return (domain, rest)
|
||||
|
||||
def urlsplit(url, scheme='', allow_fragments=True):
|
||||
"""Parse a URL into 5 components:
|
||||
<scheme>://<netloc>/<path>?<query>#<fragment>
|
||||
Return a 5-tuple: (scheme, netloc, path, query, fragment).
|
||||
Note that we don't break the components up in smaller bits
|
||||
(e.g. netloc is a single string) and we don't expand % escapes."""
|
||||
url, scheme, _coerce_result = _coerce_args(url, scheme)
|
||||
allow_fragments = bool(allow_fragments)
|
||||
key = url, scheme, allow_fragments, type(url), type(scheme)
|
||||
cached = _parse_cache.get(key, None)
|
||||
if cached:
|
||||
return _coerce_result(cached)
|
||||
if len(_parse_cache) >= MAX_CACHE_SIZE: # avoid runaway growth
|
||||
clear_cache()
|
||||
netloc = query = fragment = ''
|
||||
i = url.find(':')
|
||||
if i > 0:
|
||||
if url[:i] == 'http': # optimize the common case
|
||||
scheme = url[:i].lower()
|
||||
url = url[i+1:]
|
||||
if url[:2] == '//':
|
||||
netloc, url = _splitnetloc(url, 2)
|
||||
if (('[' in netloc and ']' not in netloc) or
|
||||
(']' in netloc and '[' not in netloc)):
|
||||
raise ValueError("Invalid IPv6 URL")
|
||||
if allow_fragments and '#' in url:
|
||||
url, fragment = url.split('#', 1)
|
||||
if '?' in url:
|
||||
url, query = url.split('?', 1)
|
||||
v = SplitResult(scheme, netloc, url, query, fragment)
|
||||
_parse_cache[key] = v
|
||||
return _coerce_result(v)
|
||||
for c in url[:i]:
|
||||
if c not in scheme_chars:
|
||||
break
|
||||
else:
|
||||
# make sure "url" is not actually a port number (in which case
|
||||
# "scheme" is really part of the path)
|
||||
rest = url[i+1:]
|
||||
if not rest or any(c not in '0123456789' for c in rest):
|
||||
# not a port number
|
||||
scheme, url = url[:i].lower(), rest
|
||||
|
||||
if url[:2] == '//':
|
||||
netloc, url = _splitnetloc(url, 2)
|
||||
if (('[' in netloc and ']' not in netloc) or
|
||||
(']' in netloc and '[' not in netloc)):
|
||||
raise ValueError("Invalid IPv6 URL")
|
||||
if allow_fragments and '#' in url:
|
||||
url, fragment = url.split('#', 1)
|
||||
if '?' in url:
|
||||
url, query = url.split('?', 1)
|
||||
v = SplitResult(scheme, netloc, url, query, fragment)
|
||||
_parse_cache[key] = v
|
||||
return _coerce_result(v)
|
||||
|
||||
def urlunparse(components):
|
||||
"""Put a parsed URL back together again. This may result in a
|
||||
slightly different, but equivalent URL, if the URL that was parsed
|
||||
originally had redundant delimiters, e.g. a ? with an empty query
|
||||
(the draft states that these are equivalent)."""
|
||||
scheme, netloc, url, params, query, fragment, _coerce_result = (
|
||||
_coerce_args(*components))
|
||||
if params:
|
||||
url = "%s;%s" % (url, params)
|
||||
return _coerce_result(urlunsplit((scheme, netloc, url, query, fragment)))
|
||||
|
||||
def urlunsplit(components):
|
||||
"""Combine the elements of a tuple as returned by urlsplit() into a
|
||||
complete URL as a string. The data argument can be any five-item iterable.
|
||||
This may result in a slightly different, but equivalent URL, if the URL that
|
||||
was parsed originally had unnecessary delimiters (for example, a ? with an
|
||||
empty query; the RFC states that these are equivalent)."""
|
||||
scheme, netloc, url, query, fragment, _coerce_result = (
|
||||
_coerce_args(*components))
|
||||
if netloc or (scheme and scheme in uses_netloc and url[:2] != '//'):
|
||||
if url and url[:1] != '/': url = '/' + url
|
||||
url = '//' + (netloc or '') + url
|
||||
if scheme:
|
||||
url = scheme + ':' + url
|
||||
if query:
|
||||
url = url + '?' + query
|
||||
if fragment:
|
||||
url = url + '#' + fragment
|
||||
return _coerce_result(url)
|
||||
|
||||
def urljoin(base, url, allow_fragments=True):
|
||||
"""Join a base URL and a possibly relative URL to form an absolute
|
||||
interpretation of the latter."""
|
||||
if not base:
|
||||
return url
|
||||
if not url:
|
||||
return base
|
||||
base, url, _coerce_result = _coerce_args(base, url)
|
||||
bscheme, bnetloc, bpath, bparams, bquery, bfragment = \
|
||||
urlparse(base, '', allow_fragments)
|
||||
scheme, netloc, path, params, query, fragment = \
|
||||
urlparse(url, bscheme, allow_fragments)
|
||||
if scheme != bscheme or scheme not in uses_relative:
|
||||
return _coerce_result(url)
|
||||
if scheme in uses_netloc:
|
||||
if netloc:
|
||||
return _coerce_result(urlunparse((scheme, netloc, path,
|
||||
params, query, fragment)))
|
||||
netloc = bnetloc
|
||||
if path[:1] == '/':
|
||||
return _coerce_result(urlunparse((scheme, netloc, path,
|
||||
params, query, fragment)))
|
||||
if not path and not params:
|
||||
path = bpath
|
||||
params = bparams
|
||||
if not query:
|
||||
query = bquery
|
||||
return _coerce_result(urlunparse((scheme, netloc, path,
|
||||
params, query, fragment)))
|
||||
segments = bpath.split('/')[:-1] + path.split('/')
|
||||
# XXX The stuff below is bogus in various ways...
|
||||
if segments[-1] == '.':
|
||||
segments[-1] = ''
|
||||
while '.' in segments:
|
||||
segments.remove('.')
|
||||
while 1:
|
||||
i = 1
|
||||
n = len(segments) - 1
|
||||
while i < n:
|
||||
if (segments[i] == '..'
|
||||
and segments[i-1] not in ('', '..')):
|
||||
del segments[i-1:i+1]
|
||||
break
|
||||
i = i+1
|
||||
else:
|
||||
break
|
||||
if segments == ['', '..']:
|
||||
segments[-1] = ''
|
||||
elif len(segments) >= 2 and segments[-1] == '..':
|
||||
segments[-2:] = ['']
|
||||
return _coerce_result(urlunparse((scheme, netloc, '/'.join(segments),
|
||||
params, query, fragment)))
|
||||
|
||||
def urldefrag(url):
|
||||
"""Removes any existing fragment from URL.
|
||||
|
||||
Returns a tuple of the defragmented URL and the fragment. If
|
||||
the URL contained no fragments, the second element is the
|
||||
empty string.
|
||||
"""
|
||||
url, _coerce_result = _coerce_args(url)
|
||||
if '#' in url:
|
||||
s, n, p, a, q, frag = urlparse(url)
|
||||
defrag = urlunparse((s, n, p, a, q, ''))
|
||||
else:
|
||||
frag = ''
|
||||
defrag = url
|
||||
return _coerce_result(DefragResult(defrag, frag))
|
||||
|
||||
_hexdig = '0123456789ABCDEFabcdef'
|
||||
_hextobyte = {(a + b).encode(): bytes([int(a + b, 16)])
|
||||
for a in _hexdig for b in _hexdig}
|
||||
|
||||
def unquote_to_bytes(string):
|
||||
"""unquote_to_bytes('abc%20def') -> b'abc def'."""
|
||||
# Note: strings are encoded as UTF-8. This is only an issue if it contains
|
||||
# unescaped non-ASCII characters, which URIs should not.
|
||||
if not string:
|
||||
# Is it a string-like object?
|
||||
string.split
|
||||
return b''
|
||||
if isinstance(string, str):
|
||||
string = string.encode('utf-8')
|
||||
bits = string.split(b'%')
|
||||
if len(bits) == 1:
|
||||
return string
|
||||
res = [bits[0]]
|
||||
append = res.append
|
||||
for item in bits[1:]:
|
||||
try:
|
||||
append(_hextobyte[item[:2]])
|
||||
append(item[2:])
|
||||
except KeyError:
|
||||
append(b'%')
|
||||
append(item)
|
||||
return b''.join(res)
|
||||
|
||||
_asciire = re.compile('([\x00-\x7f]+)')
|
||||
|
||||
def unquote(string, encoding='utf-8', errors='replace'):
|
||||
"""Replace %xx escapes by their single-character equivalent. The optional
|
||||
encoding and errors parameters specify how to decode percent-encoded
|
||||
sequences into Unicode characters, as accepted by the bytes.decode()
|
||||
method.
|
||||
By default, percent-encoded sequences are decoded with UTF-8, and invalid
|
||||
sequences are replaced by a placeholder character.
|
||||
|
||||
unquote('abc%20def') -> 'abc def'.
|
||||
"""
|
||||
if '%' not in string:
|
||||
string.split
|
||||
return string
|
||||
if encoding is None:
|
||||
encoding = 'utf-8'
|
||||
if errors is None:
|
||||
errors = 'replace'
|
||||
bits = _asciire.split(string)
|
||||
res = [bits[0]]
|
||||
append = res.append
|
||||
for i in range(1, len(bits), 2):
|
||||
append(unquote_to_bytes(bits[i]).decode(encoding, errors))
|
||||
append(bits[i + 1])
|
||||
return ''.join(res)
|
||||
|
||||
def parse_qs(qs, keep_blank_values=False, strict_parsing=False,
|
||||
encoding='utf-8', errors='replace'):
|
||||
"""Parse a query given as a string argument.
|
||||
|
||||
Arguments:
|
||||
|
||||
qs: percent-encoded query string to be parsed
|
||||
|
||||
keep_blank_values: flag indicating whether blank values in
|
||||
percent-encoded queries should be treated as blank strings.
|
||||
A true value indicates that blanks should be retained as
|
||||
blank strings. The default false value indicates that
|
||||
blank values are to be ignored and treated as if they were
|
||||
not included.
|
||||
|
||||
strict_parsing: flag indicating what to do with parsing errors.
|
||||
If false (the default), errors are silently ignored.
|
||||
If true, errors raise a ValueError exception.
|
||||
|
||||
encoding and errors: specify how to decode percent-encoded sequences
|
||||
into Unicode characters, as accepted by the bytes.decode() method.
|
||||
"""
|
||||
parsed_result = {}
|
||||
pairs = parse_qsl(qs, keep_blank_values, strict_parsing,
|
||||
encoding=encoding, errors=errors)
|
||||
for name, value in pairs:
|
||||
if name in parsed_result:
|
||||
parsed_result[name].append(value)
|
||||
else:
|
||||
parsed_result[name] = [value]
|
||||
return parsed_result
|
||||
|
||||
def parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
|
||||
encoding='utf-8', errors='replace'):
|
||||
"""Parse a query given as a string argument.
|
||||
|
||||
Arguments:
|
||||
|
||||
qs: percent-encoded query string to be parsed
|
||||
|
||||
keep_blank_values: flag indicating whether blank values in
|
||||
percent-encoded queries should be treated as blank strings. A
|
||||
true value indicates that blanks should be retained as blank
|
||||
strings. The default false value indicates that blank values
|
||||
are to be ignored and treated as if they were not included.
|
||||
|
||||
strict_parsing: flag indicating what to do with parsing errors. If
|
||||
false (the default), errors are silently ignored. If true,
|
||||
errors raise a ValueError exception.
|
||||
|
||||
encoding and errors: specify how to decode percent-encoded sequences
|
||||
into Unicode characters, as accepted by the bytes.decode() method.
|
||||
|
||||
Returns a list, as G-d intended.
|
||||
"""
|
||||
qs, _coerce_result = _coerce_args(qs)
|
||||
pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
|
||||
r = []
|
||||
for name_value in pairs:
|
||||
if not name_value and not strict_parsing:
|
||||
continue
|
||||
nv = name_value.split('=', 1)
|
||||
if len(nv) != 2:
|
||||
if strict_parsing:
|
||||
raise ValueError("bad query field: %r" % (name_value,))
|
||||
# Handle case of a control-name with no equal sign
|
||||
if keep_blank_values:
|
||||
nv.append('')
|
||||
else:
|
||||
continue
|
||||
if len(nv[1]) or keep_blank_values:
|
||||
name = nv[0].replace('+', ' ')
|
||||
name = unquote(name, encoding=encoding, errors=errors)
|
||||
name = _coerce_result(name)
|
||||
value = nv[1].replace('+', ' ')
|
||||
value = unquote(value, encoding=encoding, errors=errors)
|
||||
value = _coerce_result(value)
|
||||
r.append((name, value))
|
||||
return r
|
||||
|
||||
def unquote_plus(string, encoding='utf-8', errors='replace'):
|
||||
"""Like unquote(), but also replace plus signs by spaces, as required for
|
||||
unquoting HTML form values.
|
||||
|
||||
unquote_plus('%7e/abc+def') -> '~/abc def'
|
||||
"""
|
||||
string = string.replace('+', ' ')
|
||||
return unquote(string, encoding, errors)
|
||||
|
||||
_ALWAYS_SAFE = frozenset(b'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
|
||||
b'abcdefghijklmnopqrstuvwxyz'
|
||||
b'0123456789'
|
||||
b'_.-')
|
||||
_ALWAYS_SAFE_BYTES = bytes(_ALWAYS_SAFE)
|
||||
_safe_quoters = {}
|
||||
|
||||
class Quoter(collections.defaultdict):
|
||||
"""A mapping from bytes (in range(0,256)) to strings.
|
||||
|
||||
String values are percent-encoded byte values, unless the key < 128, and
|
||||
in the "safe" set (either the specified safe set, or default set).
|
||||
"""
|
||||
# Keeps a cache internally, using defaultdict, for efficiency (lookups
|
||||
# of cached keys don't call Python code at all).
|
||||
def __init__(self, safe):
|
||||
"""safe: bytes object."""
|
||||
self.safe = _ALWAYS_SAFE.union(safe)
|
||||
|
||||
def __repr__(self):
|
||||
# Without this, will just display as a defaultdict
|
||||
return "<Quoter %r>" % dict(self)
|
||||
|
||||
def __missing__(self, b):
|
||||
# Handle a cache miss. Store quoted string in cache and return.
|
||||
res = chr(b) if b in self.safe else '%{:02X}'.format(b)
|
||||
self[b] = res
|
||||
return res
|
||||
|
||||
def quote(string, safe='/', encoding=None, errors=None):
|
||||
"""quote('abc def') -> 'abc%20def'
|
||||
|
||||
Each part of a URL, e.g. the path info, the query, etc., has a
|
||||
different set of reserved characters that must be quoted.
|
||||
|
||||
RFC 2396 Uniform Resource Identifiers (URI): Generic Syntax lists
|
||||
the following reserved characters.
|
||||
|
||||
reserved = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" |
|
||||
"$" | ","
|
||||
|
||||
Each of these characters is reserved in some component of a URL,
|
||||
but not necessarily in all of them.
|
||||
|
||||
By default, the quote function is intended for quoting the path
|
||||
section of a URL. Thus, it will not encode '/'. This character
|
||||
is reserved, but in typical usage the quote function is being
|
||||
called on a path where the existing slash characters are used as
|
||||
reserved characters.
|
||||
|
||||
string and safe may be either str or bytes objects. encoding must
|
||||
not be specified if string is a str.
|
||||
|
||||
The optional encoding and errors parameters specify how to deal with
|
||||
non-ASCII characters, as accepted by the str.encode method.
|
||||
By default, encoding='utf-8' (characters are encoded with UTF-8), and
|
||||
errors='strict' (unsupported characters raise a UnicodeEncodeError).
|
||||
"""
|
||||
if isinstance(string, str):
|
||||
if not string:
|
||||
return string
|
||||
if encoding is None:
|
||||
encoding = 'utf-8'
|
||||
if errors is None:
|
||||
errors = 'strict'
|
||||
string = string.encode(encoding, errors)
|
||||
else:
|
||||
if encoding is not None:
|
||||
raise TypeError("quote() doesn't support 'encoding' for bytes")
|
||||
if errors is not None:
|
||||
raise TypeError("quote() doesn't support 'errors' for bytes")
|
||||
return quote_from_bytes(string, safe)
|
||||
|
||||
def quote_plus(string, safe='', encoding=None, errors=None):
|
||||
"""Like quote(), but also replace ' ' with '+', as required for quoting
|
||||
HTML form values. Plus signs in the original string are escaped unless
|
||||
they are included in safe. It also does not have safe default to '/'.
|
||||
"""
|
||||
# Check if ' ' in string, where string may either be a str or bytes. If
|
||||
# there are no spaces, the regular quote will produce the right answer.
|
||||
if ((isinstance(string, str) and ' ' not in string) or
|
||||
(isinstance(string, bytes) and b' ' not in string)):
|
||||
return quote(string, safe, encoding, errors)
|
||||
if isinstance(safe, str):
|
||||
space = ' '
|
||||
else:
|
||||
space = b' '
|
||||
string = quote(string, safe + space, encoding, errors)
|
||||
return string.replace(' ', '+')
|
||||
|
||||
def quote_from_bytes(bs, safe='/'):
|
||||
"""Like quote(), but accepts a bytes object rather than a str, and does
|
||||
not perform string-to-bytes encoding. It always returns an ASCII string.
|
||||
quote_from_bytes(b'abc def\x3f') -> 'abc%20def%3f'
|
||||
"""
|
||||
if not isinstance(bs, (bytes, bytearray)):
|
||||
raise TypeError("quote_from_bytes() expected bytes")
|
||||
if not bs:
|
||||
return ''
|
||||
if isinstance(safe, str):
|
||||
# Normalize 'safe' by converting to bytes and removing non-ASCII chars
|
||||
safe = safe.encode('ascii', 'ignore')
|
||||
else:
|
||||
safe = bytes([c for c in safe if c < 128])
|
||||
if not bs.rstrip(_ALWAYS_SAFE_BYTES + safe):
|
||||
return bs.decode()
|
||||
try:
|
||||
quoter = _safe_quoters[safe]
|
||||
except KeyError:
|
||||
_safe_quoters[safe] = quoter = Quoter(safe).__getitem__
|
||||
return ''.join([quoter(char) for char in bs])
|
||||
|
||||
def urlencode(query, doseq=False, safe='', encoding=None, errors=None):
|
||||
"""Encode a dict or sequence of two-element tuples into a URL query string.
|
||||
|
||||
If any values in the query arg are sequences and doseq is true, each
|
||||
sequence element is converted to a separate parameter.
|
||||
|
||||
If the query arg is a sequence of two-element tuples, the order of the
|
||||
parameters in the output will match the order of parameters in the
|
||||
input.
|
||||
|
||||
The components of a query arg may each be either a string or a bytes type.
|
||||
When a component is a string, the safe, encoding and error parameters are
|
||||
sent to the quote_plus function for encoding.
|
||||
"""
|
||||
|
||||
if hasattr(query, "items"):
|
||||
query = query.items()
|
||||
else:
|
||||
# It's a bother at times that strings and string-like objects are
|
||||
# sequences.
|
||||
try:
|
||||
# non-sequence items should not work with len()
|
||||
# non-empty strings will fail this
|
||||
if len(query) and not isinstance(query[0], tuple):
|
||||
raise TypeError
|
||||
# Zero-length sequences of all types will get here and succeed,
|
||||
# but that's a minor nit. Since the original implementation
|
||||
# allowed empty dicts that type of behavior probably should be
|
||||
# preserved for consistency
|
||||
except TypeError:
|
||||
ty, va, tb = sys.exc_info()
|
||||
raise TypeError("not a valid non-string sequence "
|
||||
"or mapping object").with_traceback(tb)
|
||||
|
||||
l = []
|
||||
if not doseq:
|
||||
for k, v in query:
|
||||
if isinstance(k, bytes):
|
||||
k = quote_plus(k, safe)
|
||||
else:
|
||||
k = quote_plus(str(k), safe, encoding, errors)
|
||||
|
||||
if isinstance(v, bytes):
|
||||
v = quote_plus(v, safe)
|
||||
else:
|
||||
v = quote_plus(str(v), safe, encoding, errors)
|
||||
l.append(k + '=' + v)
|
||||
else:
|
||||
for k, v in query:
|
||||
if isinstance(k, bytes):
|
||||
k = quote_plus(k, safe)
|
||||
else:
|
||||
k = quote_plus(str(k), safe, encoding, errors)
|
||||
|
||||
if isinstance(v, bytes):
|
||||
v = quote_plus(v, safe)
|
||||
l.append(k + '=' + v)
|
||||
elif isinstance(v, str):
|
||||
v = quote_plus(v, safe, encoding, errors)
|
||||
l.append(k + '=' + v)
|
||||
else:
|
||||
try:
|
||||
# Is this a sufficient test for sequence-ness?
|
||||
x = len(v)
|
||||
except TypeError:
|
||||
# not a sequence
|
||||
v = quote_plus(str(v), safe, encoding, errors)
|
||||
l.append(k + '=' + v)
|
||||
else:
|
||||
# loop over the sequence
|
||||
for elt in v:
|
||||
if isinstance(elt, bytes):
|
||||
elt = quote_plus(elt, safe)
|
||||
else:
|
||||
elt = quote_plus(str(elt), safe, encoding, errors)
|
||||
l.append(k + '=' + elt)
|
||||
return '&'.join(l)
|
||||
|
||||
# Utilities to parse URLs (most of these return None for missing parts):
|
||||
# unwrap('<URL:type://host/path>') --> 'type://host/path'
|
||||
# splittype('type:opaquestring') --> 'type', 'opaquestring'
|
||||
# splithost('//host[:port]/path') --> 'host[:port]', '/path'
|
||||
# splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'
|
||||
# splitpasswd('user:passwd') -> 'user', 'passwd'
|
||||
# splitport('host:port') --> 'host', 'port'
|
||||
# splitquery('/path?query') --> '/path', 'query'
|
||||
# splittag('/path#tag') --> '/path', 'tag'
|
||||
# splitattr('/path;attr1=value1;attr2=value2;...') ->
|
||||
# '/path', ['attr1=value1', 'attr2=value2', ...]
|
||||
# splitvalue('attr=value') --> 'attr', 'value'
|
||||
# urllib.parse.unquote('abc%20def') -> 'abc def'
|
||||
# quote('abc def') -> 'abc%20def')
|
||||
|
||||
def to_bytes(url):
|
||||
"""to_bytes(u"URL") --> 'URL'."""
|
||||
# Most URL schemes require ASCII. If that changes, the conversion
|
||||
# can be relaxed.
|
||||
# XXX get rid of to_bytes()
|
||||
if isinstance(url, str):
|
||||
try:
|
||||
url = url.encode("ASCII").decode()
|
||||
except UnicodeError:
|
||||
raise UnicodeError("URL " + repr(url) +
|
||||
" contains non-ASCII characters")
|
||||
return url
|
||||
|
||||
def unwrap(url):
|
||||
"""unwrap('<URL:type://host/path>') --> 'type://host/path'."""
|
||||
url = str(url).strip()
|
||||
if url[:1] == '<' and url[-1:] == '>':
|
||||
url = url[1:-1].strip()
|
||||
if url[:4] == 'URL:': url = url[4:].strip()
|
||||
return url
|
||||
|
||||
_typeprog = None
|
||||
def splittype(url):
|
||||
"""splittype('type:opaquestring') --> 'type', 'opaquestring'."""
|
||||
global _typeprog
|
||||
if _typeprog is None:
|
||||
import re
|
||||
_typeprog = re.compile('^([^/:]+):')
|
||||
|
||||
match = _typeprog.match(url)
|
||||
if match:
|
||||
scheme = match.group(1)
|
||||
return scheme.lower(), url[len(scheme) + 1:]
|
||||
return None, url
|
||||
|
||||
_hostprog = None
|
||||
def splithost(url):
|
||||
"""splithost('//host[:port]/path') --> 'host[:port]', '/path'."""
|
||||
global _hostprog
|
||||
if _hostprog is None:
|
||||
import re
|
||||
_hostprog = re.compile('^//([^/?]*)(.*)$')
|
||||
|
||||
match = _hostprog.match(url)
|
||||
if match:
|
||||
host_port = match.group(1)
|
||||
path = match.group(2)
|
||||
if path and not path.startswith('/'):
|
||||
path = '/' + path
|
||||
return host_port, path
|
||||
return None, url
|
||||
|
||||
_userprog = None
|
||||
def splituser(host):
|
||||
"""splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'."""
|
||||
global _userprog
|
||||
if _userprog is None:
|
||||
import re
|
||||
_userprog = re.compile('^(.*)@(.*)$')
|
||||
|
||||
match = _userprog.match(host)
|
||||
if match: return match.group(1, 2)
|
||||
return None, host
|
||||
|
||||
_passwdprog = None
|
||||
def splitpasswd(user):
|
||||
"""splitpasswd('user:passwd') -> 'user', 'passwd'."""
|
||||
global _passwdprog
|
||||
if _passwdprog is None:
|
||||
import re
|
||||
_passwdprog = re.compile('^([^:]*):(.*)$',re.S)
|
||||
|
||||
match = _passwdprog.match(user)
|
||||
if match: return match.group(1, 2)
|
||||
return user, None
|
||||
|
||||
# splittag('/path#tag') --> '/path', 'tag'
|
||||
_portprog = None
|
||||
def splitport(host):
|
||||
"""splitport('host:port') --> 'host', 'port'."""
|
||||
global _portprog
|
||||
if _portprog is None:
|
||||
import re
|
||||
_portprog = re.compile('^(.*):([0-9]+)$')
|
||||
|
||||
match = _portprog.match(host)
|
||||
if match: return match.group(1, 2)
|
||||
return host, None
|
||||
|
||||
_nportprog = None
|
||||
def splitnport(host, defport=-1):
|
||||
"""Split host and port, returning numeric port.
|
||||
Return given default port if no ':' found; defaults to -1.
|
||||
Return numerical port if a valid number are found after ':'.
|
||||
Return None if ':' but not a valid number."""
|
||||
global _nportprog
|
||||
if _nportprog is None:
|
||||
import re
|
||||
_nportprog = re.compile('^(.*):(.*)$')
|
||||
|
||||
match = _nportprog.match(host)
|
||||
if match:
|
||||
host, port = match.group(1, 2)
|
||||
try:
|
||||
if not port: raise ValueError("no digits")
|
||||
nport = int(port)
|
||||
except ValueError:
|
||||
nport = None
|
||||
return host, nport
|
||||
return host, defport
|
||||
|
||||
_queryprog = None
|
||||
def splitquery(url):
|
||||
"""splitquery('/path?query') --> '/path', 'query'."""
|
||||
global _queryprog
|
||||
if _queryprog is None:
|
||||
import re
|
||||
_queryprog = re.compile('^(.*)\?([^?]*)$')
|
||||
|
||||
match = _queryprog.match(url)
|
||||
if match: return match.group(1, 2)
|
||||
return url, None
|
||||
|
||||
_tagprog = None
|
||||
def splittag(url):
|
||||
"""splittag('/path#tag') --> '/path', 'tag'."""
|
||||
global _tagprog
|
||||
if _tagprog is None:
|
||||
import re
|
||||
_tagprog = re.compile('^(.*)#([^#]*)$')
|
||||
|
||||
match = _tagprog.match(url)
|
||||
if match: return match.group(1, 2)
|
||||
return url, None
|
||||
|
||||
def splitattr(url):
|
||||
"""splitattr('/path;attr1=value1;attr2=value2;...') ->
|
||||
'/path', ['attr1=value1', 'attr2=value2', ...]."""
|
||||
words = url.split(';')
|
||||
return words[0], words[1:]
|
||||
|
||||
_valueprog = None
|
||||
def splitvalue(attr):
|
||||
"""splitvalue('attr=value') --> 'attr', 'value'."""
|
||||
global _valueprog
|
||||
if _valueprog is None:
|
||||
import re
|
||||
_valueprog = re.compile('^([^=]*)=(.*)$')
|
||||
|
||||
match = _valueprog.match(attr)
|
||||
if match: return match.group(1, 2)
|
||||
return attr, None
|
||||
Ładowanie…
Reference in New Issue