From c469d4c3aa0a66579d1927f0e5d9630b3ea4024f Mon Sep 17 00:00:00 2001 From: Facundo Batista Date: Wed, 3 Sep 2008 22:49:01 +0000 Subject: [PATCH] Issue 600362: Relocated parse_qs() and parse_qsl(), from the cgi module to the urlparse one. Added a DeprecationWarning in the old module, it will be deprecated in the future. Docs and tests updated. --- Doc/library/cgi.rst | 43 +++--------------- Doc/library/urllib.parse.rst | 43 +++++++++++++++++- Lib/cgi.py | 85 ++++++++---------------------------- Lib/test/test_cgi.py | 24 ---------- Lib/test/test_urlparse.py | 23 ++++++++++ Lib/urllib/parse.py | 68 ++++++++++++++++++++++++++++- Misc/NEWS | 4 ++ 7 files changed, 159 insertions(+), 131 deletions(-) diff --git a/Doc/library/cgi.rst b/Doc/library/cgi.rst index fd1e92ff110..c8d9903bc56 100644 --- a/Doc/library/cgi.rst +++ b/Doc/library/cgi.rst @@ -257,49 +257,18 @@ algorithms implemented in this module in other circumstances. Parse a query in the environment or from a file (the file defaults to ``sys.stdin``). The *keep_blank_values* and *strict_parsing* parameters are - passed to :func:`parse_qs` unchanged. + passed to :func:`urllib.parse.parse_qs` unchanged. .. function:: parse_qs(qs[, keep_blank_values[, strict_parsing]]) - Parse a query string given as a string argument (data of type - :mimetype:`application/x-www-form-urlencoded`). Data are returned as a - dictionary. The dictionary keys are the unique query variable names and the - values are lists of values for each name. - - The optional argument *keep_blank_values* is a flag indicating whether blank - values in URL encoded queries should be treated as blank strings. A true value - indicates that blanks should be retained as blank strings. The default false - value indicates that blank values are to be ignored and treated as if they were - not included. - - The optional argument *strict_parsing* is a flag indicating what to do with - parsing errors. If false (the default), errors are silently ignored. If true, - errors raise a :exc:`ValueError` exception. - - Use the :func:`urllib.parse.urlencode` function to convert such dictionaries into - query strings. - + This function is deprecated in this module. Use :func:`urllib.parse.parse_qs` + instead. It is maintained here only for backward compatiblity. .. function:: parse_qsl(qs[, keep_blank_values[, strict_parsing]]) - Parse a query string given as a string argument (data of type - :mimetype:`application/x-www-form-urlencoded`). Data are returned as a list of - name, value pairs. - - The optional argument *keep_blank_values* is a flag indicating whether blank - values in URL encoded queries should be treated as blank strings. A true value - indicates that blanks should be retained as blank strings. The default false - value indicates that blank values are to be ignored and treated as if they were - not included. - - The optional argument *strict_parsing* is a flag indicating what to do with - parsing errors. If false (the default), errors are silently ignored. If true, - errors raise a :exc:`ValueError` exception. - - Use the :func:`urllib.parse.urlencode` function to convert such lists of pairs into - query strings. - + This function is deprecated in this module. Use :func:`urllib.parse.parse_qs` + instead. It is maintained here only for backward compatiblity. .. function:: parse_multipart(fp, pdict) @@ -307,7 +276,7 @@ algorithms implemented in this module in other circumstances. Arguments are *fp* for the input file and *pdict* for a dictionary containing other parameters in the :mailheader:`Content-Type` header. - Returns a dictionary just like :func:`parse_qs` keys are the field names, each + Returns a dictionary just like :func:`urllib.parse.parse_qs` keys are the field names, each value is a list of values for that field. This is easy to use but not much good if you are expecting megabytes to be uploaded --- in that case, use the :class:`FieldStorage` class instead which is much more flexible. diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 0848857bc26..26376e722ef 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -89,6 +89,47 @@ The :mod:`urllib.parse` module defines the following functions: object. +.. function:: parse_qs(qs[, keep_blank_values[, strict_parsing]]) + + Parse a query string given as a string argument (data of type + :mimetype:`application/x-www-form-urlencoded`). Data are returned as a + dictionary. The dictionary keys are the unique query variable names and the + values are lists of values for each name. + + The optional argument *keep_blank_values* is a flag indicating whether blank + values in URL encoded queries should be treated as blank strings. A true value + indicates that blanks should be retained as blank strings. The default false + value indicates that blank values are to be ignored and treated as if they were + not included. + + The optional argument *strict_parsing* is a flag indicating what to do with + parsing errors. If false (the default), errors are silently ignored. If true, + errors raise a :exc:`ValueError` exception. + + Use the :func:`urllib.urlencode` function to convert such dictionaries into + query strings. + + +.. function:: parse_qsl(qs[, keep_blank_values[, strict_parsing]]) + + Parse a query string given as a string argument (data of type + :mimetype:`application/x-www-form-urlencoded`). Data are returned as a list of + name, value pairs. + + The optional argument *keep_blank_values* is a flag indicating whether blank + values in URL encoded queries should be treated as blank strings. A true value + indicates that blanks should be retained as blank strings. The default false + value indicates that blank values are to be ignored and treated as if they were + not included. + + The optional argument *strict_parsing* is a flag indicating what to do with + parsing errors. If false (the default), errors are silently ignored. If true, + errors raise a :exc:`ValueError` exception. + + Use the :func:`urllib.parse.urlencode` function to convert such lists of pairs into + query strings. + + .. function:: urlunparse(parts) Construct a URL from a tuple as returned by ``urlparse()``. The *parts* @@ -273,7 +314,7 @@ The :mod:`urllib.parse` module defines the following functions: of the sequence. When a sequence of two-element tuples is used as the *query* argument, the first element of each tuple is a key and the second is a value. The order of parameters in the encoded string will match the order of parameter - tuples in the sequence. The :mod:`cgi` module provides the functions + tuples in the sequence. This module provides the functions :func:`parse_qs` and :func:`parse_qsl` which are used to parse query strings into Python data structures. diff --git a/Lib/cgi.py b/Lib/cgi.py index bc60c1d8489..d1ac4be7026 100755 --- a/Lib/cgi.py +++ b/Lib/cgi.py @@ -37,6 +37,7 @@ import sys import os import urllib.parse import email.parser +from warnings import warn __all__ = ["MiniFieldStorage", "FieldStorage", "parse", "parse_qs", "parse_qsl", "parse_multipart", @@ -153,75 +154,23 @@ def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0): else: qs = "" environ['QUERY_STRING'] = qs # XXX Shouldn't, really - return parse_qs(qs, keep_blank_values, strict_parsing) + return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing) +# parse query string function called from urlparse, +# this is done in order to maintain backward compatiblity. + def parse_qs(qs, keep_blank_values=0, strict_parsing=0): - """Parse a query given as a string argument. - - Arguments: - - qs: URL-encoded query string to be parsed - - keep_blank_values: flag indicating whether blank values in - URL encoded queries should be treated as blank strings. - A true value indicates that blanks should be retained as - blank strings. The default false value indicates that - blank values are to be ignored and treated as if they were - not included. - - strict_parsing: flag indicating what to do with parsing errors. - If false (the default), errors are silently ignored. - If true, errors raise a ValueError exception. - """ - dict = {} - for name, value in parse_qsl(qs, keep_blank_values, strict_parsing): - if name in dict: - dict[name].append(value) - else: - dict[name] = [value] - return dict + """Parse a query given as a string argument.""" + warn("cgi.parse_qs is deprecated, use urllib.parse.parse_qs instead", + DeprecationWarning) + return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing) def parse_qsl(qs, keep_blank_values=0, strict_parsing=0): - """Parse a query given as a string argument. - - Arguments: - - qs: URL-encoded query string to be parsed - - keep_blank_values: flag indicating whether blank values in - URL encoded queries should be treated as blank strings. A - true value indicates that blanks should be retained as blank - strings. The default false value indicates that blank values - are to be ignored and treated as if they were not included. - - strict_parsing: flag indicating what to do with parsing errors. If - false (the default), errors are silently ignored. If true, - errors raise a ValueError exception. - - Returns a list, as G-d intended. - """ - pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')] - r = [] - for name_value in pairs: - if not name_value and not strict_parsing: - continue - nv = name_value.split('=', 1) - if len(nv) != 2: - if strict_parsing: - raise ValueError("bad query field: %r" % (name_value,)) - # Handle case of a control-name with no equal sign - if keep_blank_values: - nv.append('') - else: - continue - if len(nv[1]) or keep_blank_values: - name = urllib.parse.unquote(nv[0].replace('+', ' ')) - value = urllib.parse.unquote(nv[1].replace('+', ' ')) - r.append((name, value)) - - return r - + """Parse a query given as a string argument.""" + warn("cgi.parse_qsl is deprecated, use urllib.parse.parse_qs instead", + DeprecationWarning) + return urllib.parse.parse_qsl(qs, keep_blank_values, strict_parsing) def parse_multipart(fp, pdict): """Parse multipart input. @@ -624,8 +573,8 @@ class FieldStorage: if self.qs_on_post: qs += '&' + self.qs_on_post self.list = list = [] - for key, value in parse_qsl(qs, self.keep_blank_values, - self.strict_parsing): + for key, value in urllib.parse.parse_qsl(qs, self.keep_blank_values, + self.strict_parsing): list.append(MiniFieldStorage(key, value)) self.skip_lines() @@ -638,8 +587,8 @@ class FieldStorage: raise ValueError('Invalid boundary in multipart form: %r' % (ib,)) self.list = [] if self.qs_on_post: - for key, value in parse_qsl(self.qs_on_post, self.keep_blank_values, - self.strict_parsing): + for key, value in urllib.parse.parse_qsl(self.qs_on_post, + self.keep_blank_values, self.strict_parsing): self.list.append(MiniFieldStorage(key, value)) FieldStorageClass = None diff --git a/Lib/test/test_cgi.py b/Lib/test/test_cgi.py index cc11acc04f3..ff2fc466532 100644 --- a/Lib/test/test_cgi.py +++ b/Lib/test/test_cgi.py @@ -53,25 +53,6 @@ def do_test(buf, method): except Exception as err: return ComparableException(err) -# A list of test cases. Each test case is a a two-tuple that contains -# a string with the query and a dictionary with the expected result. - -parse_qsl_test_cases = [ - ("", []), - ("&", []), - ("&&", []), - ("=", [('', '')]), - ("=a", [('', 'a')]), - ("a", [('a', '')]), - ("a=", [('a', '')]), - ("a=", [('a', '')]), - ("&a=b", [('a', 'b')]), - ("a=a+b&b=b+c", [('a', 'a b'), ('b', 'b c')]), - ("a=1&a=2", [('a', '1'), ('a', '2')]), - ("a=%26&b=%3D", [('a', '&'), ('b', '=')]), - ("a=%C3%BC&b=%CA%83", [('a', '\xfc'), ('b', '\u0283')]), -] - parse_strict_test_cases = [ ("", ValueError("bad query field: ''")), ("&", ValueError("bad query field: ''")), @@ -141,11 +122,6 @@ def gen_result(data, environ): class CgiTests(unittest.TestCase): - def test_qsl(self): - for orig, expect in parse_qsl_test_cases: - result = cgi.parse_qsl(orig, keep_blank_values=True) - self.assertEqual(result, expect, "Error parsing %s" % repr(orig)) - def test_strict(self): for orig, expect in parse_strict_test_cases: # Test basic parsing diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 103f89d4c5a..16bc1332f5d 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -8,6 +8,23 @@ RFC1808_BASE = "http://a/b/c/d;p?q#f" RFC2396_BASE = "http://a/b/c/d;p?q" RFC3986_BASE = "http://a/b/c/d;p?q" +# A list of test cases. Each test case is a a two-tuple that contains +# a string with the query and a dictionary with the expected result. + +parse_qsl_test_cases = [ + ("", []), + ("&", []), + ("&&", []), + ("=", [('', '')]), + ("=a", [('', 'a')]), + ("a", [('a', '')]), + ("a=", [('a', '')]), + ("a=", [('a', '')]), + ("&a=b", [('a', 'b')]), + ("a=a+b&b=b+c", [('a', 'a b'), ('b', 'b c')]), + ("a=1&a=2", [('a', '1'), ('a', '2')]), +] + class UrlParseTestCase(unittest.TestCase): def checkRoundtrips(self, url, parsed, split): @@ -61,6 +78,12 @@ class UrlParseTestCase(unittest.TestCase): self.assertEqual(result3.hostname, result.hostname) self.assertEqual(result3.port, result.port) + def test_qsl(self): + for orig, expect in parse_qsl_test_cases: + result = urllib.parse.parse_qsl(orig, keep_blank_values=True) + self.assertEqual(result, expect, "Error parsing %s" % repr(orig)) + + def test_roundtrips(self): testcases = [ ('file:///tmp/junk.txt', diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 94d77ebbf0a..9dd8981836a 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -8,7 +8,7 @@ import sys import collections __all__ = ["urlparse", "urlunparse", "urljoin", "urldefrag", - "urlsplit", "urlunsplit", + "urlsplit", "urlunsplit", "parse_qs", "parse_qsl", "quote", "quote_plus", "quote_from_bytes", "unquote", "unquote_plus", "unquote_to_bytes"] @@ -329,6 +329,72 @@ def unquote(string, encoding='utf-8', errors='replace'): res[-1] = b''.join(pct_sequence).decode(encoding, errors) return ''.join(res) +def parse_qs(qs, keep_blank_values=0, strict_parsing=0): + """Parse a query given as a string argument. + + Arguments: + + qs: URL-encoded query string to be parsed + + keep_blank_values: flag indicating whether blank values in + URL encoded queries should be treated as blank strings. + A true value indicates that blanks should be retained as + blank strings. The default false value indicates that + blank values are to be ignored and treated as if they were + not included. + + strict_parsing: flag indicating what to do with parsing errors. + If false (the default), errors are silently ignored. + If true, errors raise a ValueError exception. + """ + dict = {} + for name, value in parse_qsl(qs, keep_blank_values, strict_parsing): + if name in dict: + dict[name].append(value) + else: + dict[name] = [value] + return dict + +def parse_qsl(qs, keep_blank_values=0, strict_parsing=0): + """Parse a query given as a string argument. + + Arguments: + + qs: URL-encoded query string to be parsed + + keep_blank_values: flag indicating whether blank values in + URL encoded queries should be treated as blank strings. A + true value indicates that blanks should be retained as blank + strings. The default false value indicates that blank values + are to be ignored and treated as if they were not included. + + strict_parsing: flag indicating what to do with parsing errors. If + false (the default), errors are silently ignored. If true, + errors raise a ValueError exception. + + Returns a list, as G-d intended. + """ + pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')] + r = [] + for name_value in pairs: + if not name_value and not strict_parsing: + continue + nv = name_value.split('=', 1) + if len(nv) != 2: + if strict_parsing: + raise ValueError("bad query field: %r" % (name_value,)) + # Handle case of a control-name with no equal sign + if keep_blank_values: + nv.append('') + else: + continue + if len(nv[1]) or keep_blank_values: + name = unquote(nv[0].replace('+', ' ')) + value = unquote(nv[1].replace('+', ' ')) + r.append((name, value)) + + return r + def unquote_plus(string, encoding='utf-8', errors='replace'): """Like unquote(), but also replace plus signs by spaces, as required for unquoting HTML form values. diff --git a/Misc/NEWS b/Misc/NEWS index eca0a1706eb..16573d21964 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -73,6 +73,10 @@ C API Library ------- +- Issue 600362: Relocated parse_qs() and parse_qsl(), from the cgi module + to the urlparse one. Added a DeprecationWarning in the old module, it + will be deprecated in the future. + - The bsddb module has been removed. - Issue #3719: platform.architecture() fails if there are spaces in the