Commits

Anonymous committed 5a416a6

Make a new urllib package .

It consists of code from urllib, urllib2, urlparse, and robotparser.
The old modules have all been removed. The new package has five
submodules: urllib.parse, urllib.request, urllib.response,
urllib.error, and urllib.robotparser. The urllib.request.urlopen()
function uses the url opener from urllib2.

Note that the unittests have not been renamed for the
beta, but they will be renamed in the future.

Joint work with Senthil Kumaran.

Comments (0)

Files changed (42)

 from io import StringIO
 import sys
 import os
-import urllib
+import urllib.parse
 import email.parser
 
 __all__ = ["MiniFieldStorage", "FieldStorage",
             else:
                 continue
         if len(nv[1]) or keep_blank_values:
-            name = urllib.unquote(nv[0].replace('+', ' '))
-            value = urllib.unquote(nv[1].replace('+', ' '))
+            name = urllib.parse.unquote(nv[0].replace('+', ' '))
+            value = urllib.parse.unquote(nv[1].replace('+', ' '))
             r.append((name, value))
 
     return r

Lib/distutils/command/register.py

 
 __revision__ = "$Id$"
 
-import os, string, urllib2, getpass, urlparse
+import os, string, getpass
 import io
+import urllib.parse, urllib.request
 
 from distutils.core import PyPIRCCommand
 from distutils.errors import *
     def classifiers(self):
         ''' Fetch the list of classifiers from the server.
         '''
-        response = urllib2.urlopen(self.repository+'?:action=list_classifiers')
+        url = self.repository+'?:action=list_classifiers'
+        response = urllib.request.urlopen(url)
         print(response.read())
 
     def verify_metadata(self):
                 password = getpass.getpass('Password: ')
 
             # set up the authentication
-            auth = urllib2.HTTPPasswordMgr()
-            host = urlparse.urlparse(self.repository)[1]
+            auth = urllib.request.HTTPPasswordMgr()
+            host = urllib.parse.urlparse(self.repository)[1]
             auth.add_password(self.realm, host, username, password)
             # send the info to the server and report the result
             code, result = self.post_to_server(self.build_post_data('submit'),
             'Content-type': 'multipart/form-data; boundary=%s; charset=utf-8'%boundary,
             'Content-length': str(len(body))
         }
-        req = urllib2.Request(self.repository, body, headers)
+        req = urllib.request.Request(self.repository, body, headers)
 
         # handle HTTP and include the Basic Auth handler
-        opener = urllib2.build_opener(
-            urllib2.HTTPBasicAuthHandler(password_mgr=auth)
+        opener = urllib.request.build_opener(
+            urllib.request.HTTPBasicAuthHandler(password_mgr=auth)
         )
         data = ''
         try:
             result = opener.open(req)
-        except urllib2.HTTPError as e:
+        except urllib.error.HTTPError as e:
             if self.show_response:
                 data = e.fp.read()
             result = e.code, e.msg
-        except urllib2.URLError as e:
+        except urllib.error.URLError as e:
             result = 500, str(e)
         else:
             if self.show_response:

Lib/distutils/command/upload.py

 import configparser
 import http.client
 import base64
-import urlparse
+import urllib.parse
 
 class upload(PyPIRCCommand):
 
         self.announce("Submitting %s to %s" % (filename, self.repository), log.INFO)
 
         # build the Request
-        # We can't use urllib2 since we need to send the Basic
+        # We can't use urllib since we need to send the Basic
         # auth right with the first request
+        # TODO(jhylton): Can we fix urllib?
         schema, netloc, url, params, query, fragments = \
-            urlparse.urlparse(self.repository)
+            urllib.parse.urlparse(self.repository)
         assert not params and not query and not fragments
         if schema == 'http':
             http = http.client.HTTPConnection(netloc)

Lib/email/utils.py

 import base64
 import random
 import socket
+import urllib.parse
 import warnings
 from io import StringIO
 
     charset is given but not language, the string is encoded using the empty
     string for language.
     """
-    import urllib
-    s = urllib.quote(s, safe='')
+    s = urllib.parse.quote(s, safe='')
     if charset is None and language is None:
         return s
     if language is None:
 
     params is a sequence of 2-tuples containing (param name, string value).
     """
-    import urllib
     # Copy params so we don't mess with the original
     params = params[:]
     new_params = []
             # language specifiers at the beginning of the string.
             for num, s, encoded in continuations:
                 if encoded:
-                    s = urllib.unquote(s)
+                    s = urllib.parse.unquote(s)
                     extended = True
                 value.append(s)
             value = quote(EMPTYSTRING.join(value))

Lib/http/client.py

 import socket
 import email.parser
 import email.message
-from urlparse import urlsplit
+from urllib.parse import urlsplit
 import warnings
 
 __all__ = ["HTTPResponse", "HTTPConnection",

Lib/http/cookiejar.py

 __all__ = ['Cookie', 'CookieJar', 'CookiePolicy', 'DefaultCookiePolicy',
            'FileCookieJar', 'LWPCookieJar', 'LoadError', 'MozillaCookieJar']
 
-import re, urlparse, copy, time, urllib
+import copy
+import re
+import time
+import urllib.parse, urllib.request
 try:
     import threading as _threading
 except ImportError:
 
     """
     url = request.get_full_url()
-    host = urlparse.urlparse(url)[1]
+    host = urllib.parse.urlparse(url)[1]
     if host == "":
         host = request.get_header("Host", "")
 
 def request_path(request):
     """request-URI, as defined by RFC 2965."""
     url = request.get_full_url()
-    #scheme, netloc, path, parameters, query, frag = urlparse.urlparse(url)
-    #req_path = escape_path("".join(urlparse.urlparse(url)[2:]))
-    path, parameters, query, frag = urlparse.urlparse(url)[2:]
+    path, parameters, query, frag = urllib.parse.urlparse(url)[2:]
     if parameters:
         path = "%s;%s" % (path, parameters)
     path = escape_path(path)
-    req_path = urlparse.urlunparse(("", "", path, "", query, frag))
+    req_path = urllib.parse.urlunparse(("", "", path, "", query, frag))
     if not req_path.startswith("/"):
         # fix bad RFC 2396 absoluteURI
         req_path = "/"+req_path
     # And here, kind of: draft-fielding-uri-rfc2396bis-03
     # (And in draft IRI specification: draft-duerst-iri-05)
     # (And here, for new URI schemes: RFC 2718)
-    path = urllib.quote(path, HTTP_PATH_SAFE)
+    path = urllib.parse.quote(path, HTTP_PATH_SAFE)
     path = ESCAPED_CHAR_RE.sub(uppercase_escaped_char, path)
     return path
 
     """Collection of HTTP cookies.
 
     You may not need to know about this class: try
-    urllib2.build_opener(HTTPCookieProcessor).open(url).
-
+    urllib.request.build_opener(HTTPCookieProcessor).open(url).
     """
 
     non_word_re = re.compile(r"\W")

Lib/http/server.py

 import time
 import socket # For gethostbyaddr()
 import shutil
-import urllib
+import urllib.parse
 import select
 import mimetypes
 import posixpath
             return None
         list.sort(key=lambda a: a.lower())
         r = []
-        displaypath = cgi.escape(urllib.unquote(self.path))
+        displaypath = cgi.escape(urllib.parse.unquote(self.path))
         r.append('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
         r.append("<html>\n<title>Directory listing for %s</title>\n" % displaypath)
         r.append("<body>\n<h2>Directory listing for %s</h2>\n" % displaypath)
                 displayname = name + "@"
                 # Note: a link to a directory displays with @ and links with /
             r.append('<li><a href="%s">%s</a>\n'
-                    % (urllib.quote(linkname), cgi.escape(displayname)))
+                    % (urllib.parse.quote(linkname), cgi.escape(displayname)))
         r.append("</ul>\n<hr>\n</body>\n</html>\n")
         enc = sys.getfilesystemencoding()
         encoded = ''.join(r).encode(enc)
         # abandon query parameters
         path = path.split('?',1)[0]
         path = path.split('#',1)[0]
-        path = posixpath.normpath(urllib.unquote(path))
+        path = posixpath.normpath(urllib.parse.unquote(path))
         words = path.split('/')
         words = filter(None, words)
         path = os.getcwd()
         env['SERVER_PROTOCOL'] = self.protocol_version
         env['SERVER_PORT'] = str(self.server.server_port)
         env['REQUEST_METHOD'] = self.command
-        uqrest = urllib.unquote(rest)
+        uqrest = urllib.parse.unquote(rest)
         env['PATH_INFO'] = uqrest
         env['PATH_TRANSLATED'] = self.translate_path(uqrest)
         env['SCRIPT_NAME'] = scriptname

Lib/macurl2path.py

 
 Do not import directly; use urllib instead."""
 
-import urllib
+import urllib.parse
 import os
 
 __all__ = ["url2pathname","pathname2url"]
     #
     # XXXX The .. handling should be fixed...
     #
-    tp = urllib.splittype(pathname)[0]
+    tp = urllib.parsesplittype(pathname)[0]
     if tp and tp != 'file':
         raise RuntimeError('Cannot convert non-local URL to pathname')
     # Turn starting /// into /, an empty hostname means current host
             i = i + 1
         rv = ':' + ':'.join(components)
     # and finally unquote slashes and other funny characters
-    return urllib.unquote(rv)
+    return urllib.parseunquote(rv)
 
 def pathname2url(pathname):
     """OS-specific conversion from a file system path to a relative URL
         return '/'.join(components)
 
 def _pncomp2url(component):
-    component = urllib.quote(component[:31], safe='')  # We want to quote slashes
-    return component
+    # We want to quote slashes
+    return urllib.parsequote(component[:31], safe='')
 
 def test():
     for url in ["index.html",
 
 import os
 import posixpath
-import urllib
+import urllib.parse
 
 __all__ = [
     "guess_type","guess_extension","guess_all_extensions",
         Optional `strict' argument when False adds a bunch of commonly found,
         but non-standard types.
         """
-        scheme, url = urllib.splittype(url)
+        scheme, url = urllib.parse.splittype(url)
         if scheme == 'data':
             # syntax of data URLs:
             # dataurl   := "data:" [ mediatype ] [ ";base64" ] "," data

Lib/robotparser.py

-""" robotparser.py
-
-    Copyright (C) 2000  Bastian Kleineidam
-
-    You can choose between two licenses when using this package:
-    1) GNU GPLv2
-    2) PSF license for Python 2.2
-
-    The robots.txt Exclusion Protocol is implemented as specified in
-    http://info.webcrawler.com/mak/projects/robots/norobots-rfc.html
-"""
-import urlparse
-import urllib
-
-__all__ = ["RobotFileParser"]
-
-class RobotFileParser:
-    """ This class provides a set of methods to read, parse and answer
-    questions about a single robots.txt file.
-
-    """
-
-    def __init__(self, url=''):
-        self.entries = []
-        self.default_entry = None
-        self.disallow_all = False
-        self.allow_all = False
-        self.set_url(url)
-        self.last_checked = 0
-
-    def mtime(self):
-        """Returns the time the robots.txt file was last fetched.
-
-        This is useful for long-running web spiders that need to
-        check for new robots.txt files periodically.
-
-        """
-        return self.last_checked
-
-    def modified(self):
-        """Sets the time the robots.txt file was last fetched to the
-        current time.
-
-        """
-        import time
-        self.last_checked = time.time()
-
-    def set_url(self, url):
-        """Sets the URL referring to a robots.txt file."""
-        self.url = url
-        self.host, self.path = urlparse.urlparse(url)[1:3]
-
-    def read(self):
-        """Reads the robots.txt URL and feeds it to the parser."""
-        opener = URLopener()
-        f = opener.open(self.url)
-        lines = []
-        line = f.readline()
-        while line:
-            lines.append(line.strip())
-            line = f.readline()
-        self.errcode = opener.errcode
-        if self.errcode in (401, 403):
-            self.disallow_all = True
-        elif self.errcode >= 400:
-            self.allow_all = True
-        elif self.errcode == 200 and lines:
-            self.parse(lines)
-
-    def _add_entry(self, entry):
-        if "*" in entry.useragents:
-            # the default entry is considered last
-            self.default_entry = entry
-        else:
-            self.entries.append(entry)
-
-    def parse(self, lines):
-        """parse the input lines from a robots.txt file.
-           We allow that a user-agent: line is not preceded by
-           one or more blank lines."""
-        state = 0
-        linenumber = 0
-        entry = Entry()
-
-        for line in lines:
-            linenumber = linenumber + 1
-            if not line:
-                if state == 1:
-                    entry = Entry()
-                    state = 0
-                elif state == 2:
-                    self._add_entry(entry)
-                    entry = Entry()
-                    state = 0
-            # remove optional comment and strip line
-            i = line.find('#')
-            if i >= 0:
-                line = line[:i]
-            line = line.strip()
-            if not line:
-                continue
-            line = line.split(':', 1)
-            if len(line) == 2:
-                line[0] = line[0].strip().lower()
-                line[1] = urllib.unquote(line[1].strip())
-                if line[0] == "user-agent":
-                    if state == 2:
-                        self._add_entry(entry)
-                        entry = Entry()
-                    entry.useragents.append(line[1])
-                    state = 1
-                elif line[0] == "disallow":
-                    if state != 0:
-                        entry.rulelines.append(RuleLine(line[1], False))
-                        state = 2
-                elif line[0] == "allow":
-                    if state != 0:
-                        entry.rulelines.append(RuleLine(line[1], True))
-        if state == 2:
-            self.entries.append(entry)
-
-
-    def can_fetch(self, useragent, url):
-        """using the parsed robots.txt decide if useragent can fetch url"""
-        if self.disallow_all:
-            return False
-        if self.allow_all:
-            return True
-        # search for given user agent matches
-        # the first match counts
-        url = urllib.quote(urlparse.urlparse(urllib.unquote(url))[2]) or "/"
-        for entry in self.entries:
-            if entry.applies_to(useragent):
-                return entry.allowance(url)
-        # try the default entry last
-        if self.default_entry:
-            return self.default_entry.allowance(url)
-        # agent not found ==> access granted
-        return True
-
-
-    def __str__(self):
-        return ''.join([str(entry) + "\n" for entry in self.entries])
-
-
-class RuleLine:
-    """A rule line is a single "Allow:" (allowance==True) or "Disallow:"
-       (allowance==False) followed by a path."""
-    def __init__(self, path, allowance):
-        if path == '' and not allowance:
-            # an empty value means allow all
-            allowance = True
-        self.path = urllib.quote(path)
-        self.allowance = allowance
-
-    def applies_to(self, filename):
-        return self.path == "*" or filename.startswith(self.path)
-
-    def __str__(self):
-        return (self.allowance and "Allow" or "Disallow") + ": " + self.path
-
-
-class Entry:
-    """An entry has one or more user-agents and zero or more rulelines"""
-    def __init__(self):
-        self.useragents = []
-        self.rulelines = []
-
-    def __str__(self):
-        ret = []
-        for agent in self.useragents:
-            ret.extend(["User-agent: ", agent, "\n"])
-        for line in self.rulelines:
-            ret.extend([str(line), "\n"])
-        return ''.join(ret)
-
-    def applies_to(self, useragent):
-        """check if this entry applies to the specified agent"""
-        # split the name token and make it lower case
-        useragent = useragent.split("/")[0].lower()
-        for agent in self.useragents:
-            if agent == '*':
-                # we have the catch-all agent
-                return True
-            agent = agent.lower()
-            if agent in useragent:
-                return True
-        return False
-
-    def allowance(self, filename):
-        """Preconditions:
-        - our agent applies to this entry
-        - filename is URL decoded"""
-        for line in self.rulelines:
-            if line.applies_to(filename):
-                return line.allowance
-        return True
-
-class URLopener(urllib.FancyURLopener):
-    def __init__(self, *args):
-        urllib.FancyURLopener.__init__(self, *args)
-        self.errcode = 200
-
-    def prompt_user_passwd(self, host, realm):
-        ## If robots.txt file is accessible only with a password,
-        ## we act as if the file wasn't there.
-        return None, None
-
-    def http_error_default(self, url, fp, errcode, errmsg, headers):
-        self.errcode = errcode
-        return urllib.FancyURLopener.http_error_default(self, url, fp, errcode,
-                                                        errmsg, headers)

Lib/test/regrtest.py

 def dash_R_cleanup(fs, ps, pic, abcs):
     import gc, copyreg
     import _strptime, linecache
-    import urlparse, urllib, urllib2, mimetypes, doctest
+    import urllib.parse, urllib.request, mimetypes, doctest
     import struct, filecmp, _abcoll
     from distutils.dir_util import _path_created
     from weakref import WeakSet
     _path_created.clear()
     re.purge()
     _strptime._regex_cache.clear()
-    urlparse.clear_cache()
-    urllib.urlcleanup()
-    urllib2.install_opener(None)
+    urllib.parse.clear_cache()
+    urllib.request.urlcleanup()
     linecache.clearcache()
     mimetypes._default_mime_types()
     filecmp._cache.clear()

Lib/test/support.py

         testcase.fail('Missing SyntaxError: "%s"' % statement)
 
 def open_urlresource(url, *args, **kw):
-    import urllib, urlparse
+    import urllib.request, urllib.parse
 
     requires('urlfetch')
-    filename = urlparse.urlparse(url)[2].split('/')[-1] # '/': it's URL!
+    filename = urllib.parse.urlparse(url)[2].split('/')[-1] # '/': it's URL!
 
     for path in [os.path.curdir, os.path.pardir]:
         fn = os.path.join(path, filename)
             return open(fn, *args, **kw)
 
     print('\tfetching %s ...' % url, file=get_original_stdout())
-    fn, _ = urllib.urlretrieve(url, filename)
+    fn, _ = urllib.request.urlretrieve(url, filename)
     return open(fn, *args, **kw)
 
 

Lib/test/test___all__.py

         self.check_all("re")
         self.check_all("reprlib")
         self.check_all("rlcompleter")
-        self.check_all("robotparser")
+        self.check_all("urllib.robotparser")
         self.check_all("sched")
         self.check_all("shelve")
         self.check_all("shlex")
         self.check_all("traceback")
         self.check_all("tty")
         self.check_all("unittest")
-        self.check_all("urllib")
-        self.check_all("urlparse")
         self.check_all("uu")
         self.check_all("warnings")
         self.check_all("wave")

Lib/test/test_http_cookiejar.py

 """Tests for http/cookiejar.py."""
 
-import re, os, time, urllib2
+import re, os, time, urllib.request
 from unittest import TestCase
 
 from test import support
 
 def _interact(cookiejar, url, set_cookie_hdrs, hdr_name):
     """Perform a single request / response cycle, returning Cookie: header."""
-    req = urllib2.Request(url)
+    req = urllib.request.Request(url)
     cookiejar.add_cookie_header(req)
     cookie_hdr = req.get_header("Cookie", "")
     headers = []
             ("http://foo/", "foo.local", True),
             ("http://foo/", ".local", True),
             ]:
-            request = urllib2.Request(url)
+            request = urllib.request.Request(url)
             r = pol.domain_return_ok(domain, request)
             if ok: self.assert_(r)
             else: self.assert_(not r)
 
     def test_request_path(self):
         # with parameters
-        req = urllib2.Request("http://www.example.com/rheum/rhaponicum;"
-                              "foo=bar;sing=song?apples=pears&spam=eggs#ni")
+        req = urllib.request.Request(
+            "http://www.example.com/rheum/rhaponicum;"
+            "foo=bar;sing=song?apples=pears&spam=eggs#ni")
         self.assertEquals(request_path(req), "/rheum/rhaponicum;"
                      "foo=bar;sing=song?apples=pears&spam=eggs#ni")
         # without parameters
-        req = urllib2.Request("http://www.example.com/rheum/rhaponicum?"
-                              "apples=pears&spam=eggs#ni")
+        req = urllib.request.Request(
+            "http://www.example.com/rheum/rhaponicum?"
+            "apples=pears&spam=eggs#ni")
         self.assertEquals(request_path(req), "/rheum/rhaponicum?"
                      "apples=pears&spam=eggs#ni")
         # missing final slash
-        req = urllib2.Request("http://www.example.com")
+        req = urllib.request.Request("http://www.example.com")
         self.assertEquals(request_path(req), "/")
 
     def test_request_port(self):
-        req = urllib2.Request("http://www.acme.com:1234/",
-                              headers={"Host": "www.acme.com:4321"})
+        req = urllib.request.Request("http://www.acme.com:1234/",
+                                     headers={"Host": "www.acme.com:4321"})
         self.assertEquals(request_port(req), "1234")
-        req = urllib2.Request("http://www.acme.com/",
-                              headers={"Host": "www.acme.com:4321"})
+        req = urllib.request.Request("http://www.acme.com/",
+                                     headers={"Host": "www.acme.com:4321"})
         self.assertEquals(request_port(req), DEFAULT_HTTP_PORT)
 
     def test_request_host(self):
         # this request is illegal (RFC2616, 14.2.3)
-        req = urllib2.Request("http://1.1.1.1/",
-                              headers={"Host": "www.acme.com:80"})
+        req = urllib.request.Request("http://1.1.1.1/",
+                                     headers={"Host": "www.acme.com:80"})
         # libwww-perl wants this response, but that seems wrong (RFC 2616,
         # section 5.2, point 1., and RFC 2965 section 1, paragraph 3)
         #self.assertEquals(request_host(req), "www.acme.com")
         self.assertEquals(request_host(req), "1.1.1.1")
-        req = urllib2.Request("http://www.acme.com/",
-                              headers={"Host": "irrelevant.com"})
+        req = urllib.request.Request("http://www.acme.com/",
+                                     headers={"Host": "irrelevant.com"})
         self.assertEquals(request_host(req), "www.acme.com")
         # not actually sure this one is valid Request object, so maybe should
         # remove test for no host in url in request_host function?
-        req = urllib2.Request("/resource.html",
-                              headers={"Host": "www.acme.com"})
+        req = urllib.request.Request("/resource.html",
+                                     headers={"Host": "www.acme.com"})
         self.assertEquals(request_host(req), "www.acme.com")
         # port shouldn't be in request-host
-        req = urllib2.Request("http://www.acme.com:2345/resource.html",
-                              headers={"Host": "www.acme.com:5432"})
+        req = urllib.request.Request("http://www.acme.com:2345/resource.html",
+                                     headers={"Host": "www.acme.com:5432"})
         self.assertEquals(request_host(req), "www.acme.com")
 
     def test_is_HDN(self):
             blocked_domains=["acme.com"],
             allowed_domains=["www.acme.com"]))
 
-        req = urllib2.Request("http://acme.com/")
+        req = urllib.request.Request("http://acme.com/")
         headers = ["Set-Cookie: CUSTOMER=WILE_E_COYOTE; path=/"]
         res = FakeResponse(headers, "http://acme.com/")
         c.extract_cookies(res, req)
         self.assertEquals(len(c), 0)
 
-        req = urllib2.Request("http://www.acme.com/")
+        req = urllib.request.Request("http://www.acme.com/")
         res = FakeResponse(headers, "http://www.acme.com/")
         c.extract_cookies(res, req)
         self.assertEquals(len(c), 1)
 
-        req = urllib2.Request("http://www.coyote.com/")
+        req = urllib.request.Request("http://www.coyote.com/")
         res = FakeResponse(headers, "http://www.coyote.com/")
         c.extract_cookies(res, req)
         self.assertEquals(len(c), 1)
 
         # set a cookie with non-allowed domain...
-        req = urllib2.Request("http://www.coyote.com/")
+        req = urllib.request.Request("http://www.coyote.com/")
         res = FakeResponse(headers, "http://www.coyote.com/")
         cookies = c.make_cookies(res, req)
         c.set_cookie(cookies[0])
         c = CookieJar(policy=pol)
         headers = ["Set-Cookie: CUSTOMER=WILE_E_COYOTE; path=/"]
 
-        req = urllib2.Request("http://www.acme.com/")
+        req = urllib.request.Request("http://www.acme.com/")
         res = FakeResponse(headers, "http://www.acme.com/")
         c.extract_cookies(res, req)
         self.assertEquals(len(c), 0)
         self.assertEquals(len(c), 1)
 
         c.clear()
-        req = urllib2.Request("http://www.roadrunner.net/")
+        req = urllib.request.Request("http://www.roadrunner.net/")
         res = FakeResponse(headers, "http://www.roadrunner.net/")
         c.extract_cookies(res, req)
         self.assertEquals(len(c), 1)
-        req = urllib2.Request("http://www.roadrunner.net/")
+        req = urllib.request.Request("http://www.roadrunner.net/")
         c.add_cookie_header(req)
         self.assert_((req.has_header("Cookie") and
                       req.has_header("Cookie2")))
         self.assertEquals(len(c), 1)
 
         # set a cookie with blocked domain...
-        req = urllib2.Request("http://www.acme.com/")
+        req = urllib.request.Request("http://www.acme.com/")
         res = FakeResponse(headers, "http://www.acme.com/")
         cookies = c.make_cookies(res, req)
         c.set_cookie(cookies[0])
         url = "http://www.acme.com"
         c = CookieJar(DefaultCookiePolicy(rfc2965=True))
         interact_2965(c, url, "foo=bar; Version=1")
-        req = urllib2.Request(url)
+        req = urllib.request.Request(url)
         self.assertEquals(len(c), 1)
         c.add_cookie_header(req)
         self.assert_(req.has_header("Cookie"))
 
         def cookiejar_from_cookie_headers(headers):
             c = CookieJar()
-            req = urllib2.Request("http://www.example.com/")
+            req = urllib.request.Request("http://www.example.com/")
             r = FakeResponse(headers, "http://www.example.com/")
             c.extract_cookies(r, req)
             return c
 
         c = CookieJar(DefaultCookiePolicy(rfc2965 = True))
 
-        #req = urllib2.Request("http://1.1.1.1/",
+        #req = urllib.request.Request("http://1.1.1.1/",
         #              headers={"Host": "www.acme.com:80"})
-        req = urllib2.Request("http://www.acme.com:80/",
+        req = urllib.request.Request("http://www.acme.com:80/",
                       headers={"Host": "www.acme.com:80"})
 
         headers.append(
         res = FakeResponse(headers, "http://www.acme.com/")
         c.extract_cookies(res, req)
 
-        req = urllib2.Request("http://www.acme.com/")
+        req = urllib.request.Request("http://www.acme.com/")
         c.add_cookie_header(req)
 
         self.assertEqual(req.get_header("Cookie"), "CUSTOMER=WILE_E_COYOTE")
         res = FakeResponse(headers, "http://www.acme.com/")
         c.extract_cookies(res, req)
 
-        req = urllib2.Request("http://www.acme.com/foo/bar")
+        req = urllib.request.Request("http://www.acme.com/foo/bar")
         c.add_cookie_header(req)
 
         h = req.get_header("Cookie")
         res = FakeResponse(headers, "http://www.acme.com")
         c.extract_cookies(res, req)
 
-        req = urllib2.Request("http://www.acme.com/")
+        req = urllib.request.Request("http://www.acme.com/")
         c.add_cookie_header(req)
 
         h = req.get_header("Cookie")
                      "CUSTOMER=WILE_E_COYOTE" in h and
                      "SHIPPING=FEDEX" not in h)
 
-        req = urllib2.Request("http://www.acme.com/foo/")
+        req = urllib.request.Request("http://www.acme.com/foo/")
         c.add_cookie_header(req)
 
         h = req.get_header("Cookie")
         c = CookieJar()
         headers = []
 
-        req = urllib2.Request("http://www.acme.com/")
+        req = urllib.request.Request("http://www.acme.com/")
         headers.append("Set-Cookie: PART_NUMBER=ROCKET_LAUNCHER_0001; path=/")
         res = FakeResponse(headers, "http://www.acme.com/")
 
         c.extract_cookies(res, req)
 
-        req = urllib2.Request("http://www.acme.com/")
+        req = urllib.request.Request("http://www.acme.com/")
         c.add_cookie_header(req)
 
         self.assertEquals(req.get_header("Cookie"),
         res = FakeResponse(headers, "http://www.acme.com/")
         c.extract_cookies(res, req)
 
-        req = urllib2.Request("http://www.acme.com/ammo")
+        req = urllib.request.Request("http://www.acme.com/ammo")
         c.add_cookie_header(req)
 
         self.assert_(re.search(r"PART_NUMBER=RIDING_ROCKET_0023;\s*"
         # Some additional Netscape cookies tests.
         c = CookieJar()
         headers = []
-        req = urllib2.Request("http://foo.bar.acme.com/foo")
+        req = urllib.request.Request("http://foo.bar.acme.com/foo")
 
         # Netscape allows a host part that contains dots
         headers.append("Set-Cookie: Customer=WILE_E_COYOTE; domain=.acme.com")
         res = FakeResponse(headers, "http://www.acme.com/foo")
         c.extract_cookies(res, req)
 
-        req = urllib2.Request("http://foo.bar.acme.com/foo")
+        req = urllib.request.Request("http://foo.bar.acme.com/foo")
         c.add_cookie_header(req)
         self.assert_(
             "PART_NUMBER=3,4" in req.get_header("Cookie") and
         c = CookieJar(DefaultCookiePolicy(rfc2965 = True))
         headers = []
 
-        req = urllib2.Request("http://www.ants.com/")
+        req = urllib.request.Request("http://www.ants.com/")
         headers.append("Set-Cookie: JSESSIONID=ABCDERANDOM123; Path=")
         res = FakeResponse(headers, "http://www.ants.com/")
         c.extract_cookies(res, req)
 
-        req = urllib2.Request("http://www.ants.com/")
+        req = urllib.request.Request("http://www.ants.com/")
         c.add_cookie_header(req)
 
         self.assertEquals(req.get_header("Cookie"),
         self.assertEquals(req.get_header("Cookie2"), '$Version="1"')
 
         # missing path in the request URI
-        req = urllib2.Request("http://www.ants.com:8080")
+        req = urllib.request.Request("http://www.ants.com:8080")
         c.add_cookie_header(req)
 
         self.assertEquals(req.get_header("Cookie"),
         # Check session cookies are deleted properly by
         # CookieJar.clear_session_cookies method
 
-        req = urllib2.Request('http://www.perlmeister.com/scripts')
+        req = urllib.request.Request('http://www.perlmeister.com/scripts')
         headers = []
         headers.append("Set-Cookie: s1=session;Path=/scripts")
         headers.append("Set-Cookie: p1=perm; Domain=.perlmeister.com;"

Lib/test/test_httpservers.py

 import sys
 import base64
 import shutil
-import urllib
+import urllib.parse
 import http.client
 import tempfile
 import threading
              (res.read(), res.getheader('Content-type'), res.status))
 
     def test_post(self):
-        params = urllib.urlencode({'spam' : 1, 'eggs' : 'python', 'bacon' : 123456})
+        params = urllib.parse.urlencode(
+            {'spam' : 1, 'eggs' : 'python', 'bacon' : 123456})
         headers = {'Content-type' : 'application/x-www-form-urlencoded'}
         res = self.request('/cgi-bin/file2.py', 'POST', params, headers)
 

Lib/test/test_importhooks.py

         i = ImpWrapper()
         sys.meta_path.append(i)
         sys.path_hooks.append(ImpWrapper)
-        mnames = ("colorsys", "urlparse", "distutils.core")
+        mnames = ("colorsys", "urllib.parse", "distutils.core")
         for mname in mnames:
             parent = mname.split(".")[0]
-            for n in list(sys.modules.keys()):
+            for n in list(sys.modules):
                 if n.startswith(parent):
                     del sys.modules[n]
         for mname in mnames:
             m = __import__(mname, globals(), locals(), ["__dummy__"])
             m.__loader__  # to make sure we actually handled the import
-        # Delete urllib from modules because urlparse was imported above.
-        # Without this hack, test_socket_ssl fails if run in this order:
-        # regrtest.py test_codecmaps_tw test_importhooks test_socket_ssl
-        try:
-            del sys.modules['urllib']
-        except KeyError:
-            pass
+##        # Delete urllib from modules because urlparse was imported above.
+##        # Without this hack, test_socket_ssl fails if run in this order:
+##        # regrtest.py test_codecmaps_tw test_importhooks test_socket_ssl
+##        try:
+##            del sys.modules['urllib']
+##        except KeyError:
+##            pass
 
 def test_main():
     support.run_unittest(ImportHooksTestCase)

Lib/test/test_pyclbr.py

         # These were once about the 10 longest modules
         cm('random', ignore=('Random',))  # from _random import Random as CoreGenerator
         cm('cgi', ignore=('log',))      # set with = in module
-        cm('urllib', ignore=('_CFNumberToInt32',
-                             '_CStringFromCFString',
-                             '_CFSetup',
-                             'getproxies_registry',
-                             'proxy_bypass_registry',
-                             'proxy_bypass_macosx_sysconf',
-                             'open_https',
-                             '_https_connection',
-                             'getproxies_macosx_sysconf',
-                             'getproxies_internetconfig',)) # not on all platforms
         cm('pickle')
         cm('aifc', ignore=('openfp',))  # set with = in module
         cm('sre_parse', ignore=('dump',)) # from sre_constants import *

Lib/test/test_robotparser.py

-import unittest, robotparser
 import io
+import unittest
+import urllib.robotparser
 from test import support
 
 class RobotTestCase(unittest.TestCase):
               agent="test_robotparser"):
 
     lines = io.StringIO(robots_txt).readlines()
-    parser = robotparser.RobotFileParser()
+    parser = urllib.robotparser.RobotFileParser()
     parser.parse(lines)
     for url in good_urls:
         tests.addTest(RobotTestCase(index, parser, url, 1, agent))
         support.requires('network')
         # whole site is password-protected.
         url = 'http://mueblesmoraleda.com'
-        parser = robotparser.RobotFileParser()
+        parser = urllib.robotparser.RobotFileParser()
         parser.set_url(url)
         parser.read()
         self.assertEqual(parser.can_fetch("*", url+"/robots.txt"), False)

Lib/test/test_ssl.py

 import time
 import os
 import pprint
-import urllib, urlparse
+import urllib.parse, urllib.request
 import shutil
 import traceback
 import asyncore
 
                 """
                 # abandon query parameters
-                path = urlparse.urlparse(path)[2]
-                path = os.path.normpath(urllib.unquote(path))
+                path = urllib.parse.urlparse(path)[2]
+                path = os.path.normpath(urllib.parse.unquote(path))
                 words = path.split('/')
                 words = filter(None, words)
                 path = self.root
                 # now fetch the same data from the HTTPS server
                 url = 'https://%s:%d/%s' % (
                     HOST, server.port, os.path.split(CERTFILE)[1])
-                f = urllib.urlopen(url)
+                f = urllib.request.urlopen(url)
                 dlen = f.info().get("content-length")
                 if dlen and (int(dlen) > 0):
                     d2 = f.read(int(dlen))

Lib/test/test_urllib.py

 """Regresssion tests for urllib"""
 
-import urllib
+import urllib.parse
+import urllib.request
 import http.client
 import email.message
 import io
         hex_repr = "0%s" % hex_repr
     return "%" + hex_repr
 
+# Shortcut for testing FancyURLopener
+_urlopener = None
+def urlopen(url, data=None, proxies=None):
+    """urlopen(url [, data]) -> open file-like object"""
+    global _urlopener
+    if proxies is not None:
+        opener = urllib.request.FancyURLopener(proxies=proxies)
+    elif not _urlopener:
+        opener = urllib.request.FancyURLopener()
+        _urlopener = opener
+    else:
+        opener = _urlopener
+    if data is None:
+        return opener.open(url)
+    else:
+        return opener.open(url, data)
+
 class urlopen_FileTests(unittest.TestCase):
     """Test urlopen() opening a temporary file.
 
     """
 
     def setUp(self):
-        """Setup of a temp file to use for testing"""
-        self.text = bytes("test_urllib: %s\n" % self.__class__.__name__, "ascii")
-        FILE = open(support.TESTFN, 'wb')
+        # Create a temp file to use for testing
+        self.text = bytes("test_urllib: %s\n" % self.__class__.__name__,
+                          "ascii")
+        f = open(support.TESTFN, 'wb')
         try:
-            FILE.write(self.text)
+            f.write(self.text)
         finally:
-            FILE.close()
+            f.close()
         self.pathname = support.TESTFN
-        self.returned_obj = urllib.urlopen("file:%s" % self.pathname)
+        self.returned_obj = urlopen("file:%s" % self.pathname)
 
     def tearDown(self):
         """Shut down the open object"""
     def test_read(self):
         self.fakehttp(b"Hello!")
         try:
-            fp = urllib.urlopen("http://python.org/")
+            fp = urlopen("http://python.org/")
             self.assertEqual(fp.readline(), b"Hello!")
             self.assertEqual(fp.readline(), b"")
             self.assertEqual(fp.geturl(), 'http://python.org/')
 Content-Type: text/html; charset=iso-8859-1
 ''')
         try:
-            self.assertRaises(IOError, urllib.urlopen, "http://python.org/")
+            self.assertRaises(IOError, urlopen, "http://python.org/")
         finally:
             self.unfakehttp()
 
         # data. (#1680230)
         self.fakehttp(b'')
         try:
-            self.assertRaises(IOError, urllib.urlopen, "http://something")
+            self.assertRaises(IOError, urlopen, "http://something")
         finally:
             self.unfakehttp()
 
             except: pass
 
     def constructLocalFileUrl(self, filePath):
-        return "file://%s" % urllib.pathname2url(os.path.abspath(filePath))
+        return "file://%s" % urllib.request.pathname2url(
+            os.path.abspath(filePath))
 
     def createNewTempFile(self, data=b""):
         """Creates a new temporary file containing the specified data,
     def test_basic(self):
         # Make sure that a local file just gets its own location returned and
         # a headers value is returned.
-        result = urllib.urlretrieve("file:%s" % support.TESTFN)
+        result = urllib.request.urlretrieve("file:%s" % support.TESTFN)
         self.assertEqual(result[0], support.TESTFN)
         self.assert_(isinstance(result[1], email.message.Message),
                      "did not get a email.message.Message instance as second "
         # Test that setting the filename argument works.
         second_temp = "%s.2" % support.TESTFN
         self.registerFileForCleanUp(second_temp)
-        result = urllib.urlretrieve(self.constructLocalFileUrl(
+        result = urllib.request.urlretrieve(self.constructLocalFileUrl(
             support.TESTFN), second_temp)
         self.assertEqual(second_temp, result[0])
         self.assert_(os.path.exists(second_temp), "copy of the file was not "
             count_holder[0] = count_holder[0] + 1
         second_temp = "%s.2" % support.TESTFN
         self.registerFileForCleanUp(second_temp)
-        urllib.urlretrieve(self.constructLocalFileUrl(support.TESTFN),
+        urllib.request.urlretrieve(
+            self.constructLocalFileUrl(support.TESTFN),
             second_temp, hooktester)
 
     def test_reporthook_0_bytes(self):
         def hooktester(count, block_size, total_size, _report=report):
             _report.append((count, block_size, total_size))
         srcFileName = self.createNewTempFile()
-        urllib.urlretrieve(self.constructLocalFileUrl(srcFileName),
+        urllib.request.urlretrieve(self.constructLocalFileUrl(srcFileName),
             support.TESTFN, hooktester)
         self.assertEqual(len(report), 1)
         self.assertEqual(report[0][2], 0)
         def hooktester(count, block_size, total_size, _report=report):
             _report.append((count, block_size, total_size))
         srcFileName = self.createNewTempFile(b"x" * 5)
-        urllib.urlretrieve(self.constructLocalFileUrl(srcFileName),
+        urllib.request.urlretrieve(self.constructLocalFileUrl(srcFileName),
             support.TESTFN, hooktester)
         self.assertEqual(len(report), 2)
         self.assertEqual(report[0][1], 8192)
         def hooktester(count, block_size, total_size, _report=report):
             _report.append((count, block_size, total_size))
         srcFileName = self.createNewTempFile(b"x" * 8193)
-        urllib.urlretrieve(self.constructLocalFileUrl(srcFileName),
+        urllib.request.urlretrieve(self.constructLocalFileUrl(srcFileName),
             support.TESTFN, hooktester)
         self.assertEqual(len(report), 3)
         self.assertEqual(report[0][1], 8192)
 class QuotingTests(unittest.TestCase):
     """Tests for urllib.quote() and urllib.quote_plus()
 
-    According to RFC 2396 ("Uniform Resource Identifiers), to escape a
-    character you write it as '%' + <2 character US-ASCII hex value>.  The Python
-    code of ``'%' + hex(ord(<character>))[2:]`` escapes a character properly.
-    Case does not matter on the hex letters.
+    According to RFC 2396 (Uniform Resource Identifiers), to escape a
+    character you write it as '%' + <2 character US-ASCII hex value>.
+    The Python code of ``'%' + hex(ord(<character>))[2:]`` escapes a
+    character properly. Case does not matter on the hex letters.
 
     The various character sets specified are:
 
                                  "abcdefghijklmnopqrstuvwxyz",
                                  "0123456789",
                                  "_.-"])
-        result = urllib.quote(do_not_quote)
+        result = urllib.parse.quote(do_not_quote)
         self.assertEqual(do_not_quote, result,
                          "using quote(): %s != %s" % (do_not_quote, result))
-        result = urllib.quote_plus(do_not_quote)
+        result = urllib.parse.quote_plus(do_not_quote)
         self.assertEqual(do_not_quote, result,
                         "using quote_plus(): %s != %s" % (do_not_quote, result))
 
     def test_default_safe(self):
         # Test '/' is default value for 'safe' parameter
-        self.assertEqual(urllib.quote.__defaults__[0], '/')
+        self.assertEqual(urllib.parse.quote.__defaults__[0], '/')
 
     def test_safe(self):
         # Test setting 'safe' parameter does what it should do
         quote_by_default = "<>"
-        result = urllib.quote(quote_by_default, safe=quote_by_default)
+        result = urllib.parse.quote(quote_by_default, safe=quote_by_default)
         self.assertEqual(quote_by_default, result,
                          "using quote(): %s != %s" % (quote_by_default, result))
-        result = urllib.quote_plus(quote_by_default, safe=quote_by_default)
+        result = urllib.parse.quote_plus(quote_by_default, safe=quote_by_default)
         self.assertEqual(quote_by_default, result,
                          "using quote_plus(): %s != %s" %
                          (quote_by_default, result))
         should_quote.append(chr(127)) # For 0x7F
         should_quote = ''.join(should_quote)
         for char in should_quote:
-            result = urllib.quote(char)
+            result = urllib.parse.quote(char)
             self.assertEqual(hexescape(char), result,
                              "using quote(): %s should be escaped to %s, not %s" %
                              (char, hexescape(char), result))
-            result = urllib.quote_plus(char)
+            result = urllib.parse.quote_plus(char)
             self.assertEqual(hexescape(char), result,
                              "using quote_plus(): "
                              "%s should be escapes to %s, not %s" %
         del should_quote
         partial_quote = "ab[]cd"
         expected = "ab%5B%5Dcd"
-        result = urllib.quote(partial_quote)
+        result = urllib.parse.quote(partial_quote)
         self.assertEqual(expected, result,
                          "using quote(): %s != %s" % (expected, result))
         self.assertEqual(expected, result,
     def test_quoting_space(self):
         # Make sure quote() and quote_plus() handle spaces as specified in
         # their unique way
-        result = urllib.quote(' ')
+        result = urllib.parse.quote(' ')
         self.assertEqual(result, hexescape(' '),
                          "using quote(): %s != %s" % (result, hexescape(' ')))
-        result = urllib.quote_plus(' ')
+        result = urllib.parse.quote_plus(' ')
         self.assertEqual(result, '+',
                          "using quote_plus(): %s != +" % result)
         given = "a b cd e f"
         expect = given.replace(' ', hexescape(' '))
-        result = urllib.quote(given)
+        result = urllib.parse.quote(given)
         self.assertEqual(expect, result,
                          "using quote(): %s != %s" % (expect, result))
         expect = given.replace(' ', '+')
-        result = urllib.quote_plus(given)
+        result = urllib.parse.quote_plus(given)
         self.assertEqual(expect, result,
                          "using quote_plus(): %s != %s" % (expect, result))
 
     def test_quoting_plus(self):
-        self.assertEqual(urllib.quote_plus('alpha+beta gamma'),
+        self.assertEqual(urllib.parse.quote_plus('alpha+beta gamma'),
                          'alpha%2Bbeta+gamma')
-        self.assertEqual(urllib.quote_plus('alpha+beta gamma', '+'),
+        self.assertEqual(urllib.parse.quote_plus('alpha+beta gamma', '+'),
                          'alpha+beta+gamma')
 
 class UnquotingTests(unittest.TestCase):
         for num in range(128):
             given = hexescape(chr(num))
             expect = chr(num)
-            result = urllib.unquote(given)
+            result = urllib.parse.unquote(given)
             self.assertEqual(expect, result,
                              "using unquote(): %s != %s" % (expect, result))
-            result = urllib.unquote_plus(given)
+            result = urllib.parse.unquote_plus(given)
             self.assertEqual(expect, result,
                              "using unquote_plus(): %s != %s" %
                              (expect, result))
             escape_list.append(given)
         escape_string = ''.join(escape_list)
         del escape_list
-        result = urllib.unquote(escape_string)
+        result = urllib.parse.unquote(escape_string)
         self.assertEqual(result.count('%'), 1,
                          "using quote(): not all characters escaped; %s" %
                          result)
-        result = urllib.unquote(escape_string)
+        result = urllib.parse.unquote(escape_string)
         self.assertEqual(result.count('%'), 1,
                          "using unquote(): not all characters escaped: "
                          "%s" % result)
         # interspersed
         given = 'ab%sd' % hexescape('c')
         expect = "abcd"
-        result = urllib.unquote(given)
+        result = urllib.parse.unquote(given)
         self.assertEqual(expect, result,
                          "using quote(): %s != %s" % (expect, result))
-        result = urllib.unquote_plus(given)
+        result = urllib.parse.unquote_plus(given)
         self.assertEqual(expect, result,
                          "using unquote_plus(): %s != %s" % (expect, result))
 
         # Test difference between unquote() and unquote_plus()
         given = "are+there+spaces..."
         expect = given
-        result = urllib.unquote(given)
+        result = urllib.parse.unquote(given)
         self.assertEqual(expect, result,
                          "using unquote(): %s != %s" % (expect, result))
         expect = given.replace('+', ' ')
-        result = urllib.unquote_plus(given)
+        result = urllib.parse.unquote_plus(given)
         self.assertEqual(expect, result,
                          "using unquote_plus(): %s != %s" % (expect, result))
 
     def test_unquote_with_unicode(self):
-        r = urllib.unquote('br%C3%BCckner_sapporo_20050930.doc')
+        r = urllib.parse.unquote('br%C3%BCckner_sapporo_20050930.doc')
         self.assertEqual(r, 'br\xc3\xbcckner_sapporo_20050930.doc')
 
 class urlencode_Tests(unittest.TestCase):
 
         """
         expect_somewhere = ["1st=1", "2nd=2", "3rd=3"]
-        result = urllib.urlencode(given)
+        result = urllib.parse.urlencode(given)
         for expected in expect_somewhere:
             self.assert_(expected in result,
                          "testing %s: %s not found in %s" %
         # Make sure keys and values are quoted using quote_plus()
         given = {"&":"="}
         expect = "%s=%s" % (hexescape('&'), hexescape('='))
-        result = urllib.urlencode(given)
+        result = urllib.parse.urlencode(given)
         self.assertEqual(expect, result)
         given = {"key name":"A bunch of pluses"}
         expect = "key+name=A+bunch+of+pluses"
-        result = urllib.urlencode(given)
+        result = urllib.parse.urlencode(given)
         self.assertEqual(expect, result)
 
     def test_doseq(self):
         # Test that passing True for 'doseq' parameter works correctly
         given = {'sequence':['1', '2', '3']}
-        expect = "sequence=%s" % urllib.quote_plus(str(['1', '2', '3']))
-        result = urllib.urlencode(given)
+        expect = "sequence=%s" % urllib.parse.quote_plus(str(['1', '2', '3']))
+        result = urllib.parse.urlencode(given)
         self.assertEqual(expect, result)
-        result = urllib.urlencode(given, True)
+        result = urllib.parse.urlencode(given, True)
         for value in given["sequence"]:
             expect = "sequence=%s" % value
             self.assert_(expect in result,
         # Make sure simple tests pass
         expected_path = os.path.join("parts", "of", "a", "path")
         expected_url = "parts/of/a/path"
-        result = urllib.pathname2url(expected_path)
+        result = urllib.request.pathname2url(expected_path)
         self.assertEqual(expected_url, result,
                          "pathname2url() failed; %s != %s" %
                          (result, expected_url))
-        result = urllib.url2pathname(expected_url)
+        result = urllib.request.url2pathname(expected_url)
         self.assertEqual(expected_path, result,
                          "url2pathame() failed; %s != %s" %
                          (result, expected_path))
         # Test automatic quoting and unquoting works for pathnam2url() and
         # url2pathname() respectively
         given = os.path.join("needs", "quot=ing", "here")
-        expect = "needs/%s/here" % urllib.quote("quot=ing")
-        result = urllib.pathname2url(given)
+        expect = "needs/%s/here" % urllib.parse.quote("quot=ing")
+        result = urllib.request.pathname2url(given)
         self.assertEqual(expect, result,
                          "pathname2url() failed; %s != %s" %
                          (expect, result))
         expect = given
-        result = urllib.url2pathname(result)
+        result = urllib.request.url2pathname(result)
         self.assertEqual(expect, result,
                          "url2pathname() failed; %s != %s" %
                          (expect, result))
         given = os.path.join("make sure", "using_quote")
-        expect = "%s/using_quote" % urllib.quote("make sure")
-        result = urllib.pathname2url(given)
+        expect = "%s/using_quote" % urllib.parse.quote("make sure")
+        result = urllib.request.pathname2url(given)
         self.assertEqual(expect, result,
                          "pathname2url() failed; %s != %s" %
                          (expect, result))
         given = "make+sure/using_unquote"
         expect = os.path.join("make+sure", "using_unquote")
-        result = urllib.url2pathname(given)
+        result = urllib.request.url2pathname(given)
         self.assertEqual(expect, result,
                          "url2pathname() failed; %s != %s" %
                          (expect, result))

Lib/test/test_urllib2.py

 import io
 import socket
 
-import urllib2
-from urllib2 import Request, OpenerDirector
+import urllib.request
+from urllib.request import Request, OpenerDirector
 
 # XXX
 # Request
     def test_trivial(self):
         # A couple trivial tests
 
-        self.assertRaises(ValueError, urllib2.urlopen, 'bogus url')
+        self.assertRaises(ValueError, urllib.request.urlopen, 'bogus url')
 
         # XXX Name hacking to get this to work on Windows.
-        fname = os.path.abspath(urllib2.__file__).replace('\\', '/')
+        fname = os.path.abspath(urllib.request.__file__).replace('\\', '/')
         if fname[1:2] == ":":
             fname = fname[2:]
         # And more hacking to get it to work on MacOS. This assumes
             fname = '/' + fname.replace(':', '/')
 
         file_url = "file://%s" % fname
-        f = urllib2.urlopen(file_url)
+        f = urllib.request.urlopen(file_url)
 
         buf = f.read()
         f.close()
 
     def test_parse_http_list(self):
-        tests = [('a,b,c', ['a', 'b', 'c']),
-                 ('path"o,l"og"i"cal, example', ['path"o,l"og"i"cal', 'example']),
-                 ('a, b, "c", "d", "e,f", g, h', ['a', 'b', '"c"', '"d"', '"e,f"', 'g', 'h']),
-                 ('a="b\\"c", d="e\\,f", g="h\\\\i"', ['a="b"c"', 'd="e,f"', 'g="h\\i"'])]
+        tests = [
+            ('a,b,c', ['a', 'b', 'c']),
+            ('path"o,l"og"i"cal, example', ['path"o,l"og"i"cal', 'example']),
+            ('a, b, "c", "d", "e,f", g, h',
+             ['a', 'b', '"c"', '"d"', '"e,f"', 'g', 'h']),
+            ('a="b\\"c", d="e\\,f", g="h\\\\i"',
+             ['a="b"c"', 'd="e,f"', 'g="h\\i"'])]
         for string, list in tests:
-            self.assertEquals(urllib2.parse_http_list(string), list)
+            self.assertEquals(urllib.request.parse_http_list(string), list)
 
 
 def test_request_headers_dict():
 
 def test_password_manager(self):
     """
-    >>> mgr = urllib2.HTTPPasswordMgr()
+    >>> mgr = urllib.request.HTTPPasswordMgr()
     >>> add = mgr.add_password
     >>> add("Some Realm", "http://example.com/", "joe", "password")
     >>> add("Some Realm", "http://example.com/ni", "ni", "ni")
 
 def test_password_manager_default_port(self):
     """
-    >>> mgr = urllib2.HTTPPasswordMgr()
+    >>> mgr = urllib.request.HTTPPasswordMgr()
     >>> add = mgr.add_password
 
     The point to note here is that we can't guess the default port if there's
             res = MockResponse(200, "OK", {}, "")
             return self.parent.error("http", args[0], res, code, "", {})
         elif action == "raise":
-            raise urllib2.URLError("blah")
+            raise urllib.error.URLError("blah")
         assert False
     def close(self): pass
     def add_parent(self, parent):
         opener.add_handler(h)
     return opener
 
-class MockHTTPHandler(urllib2.BaseHandler):
+class MockHTTPHandler(urllib.request.BaseHandler):
     # useful for testing redirections and auth
     # sends supplied headers and code as first response
     # sends 200 OK as second response
         # TypeError in real code; here, returning self from these mock
         # methods would either cause no exception, or AttributeError.
 
-        from urllib2 import URLError
+        from urllib.error import URLError
 
         o = OpenerDirector()
         meth_spec = [
             [("redirect_request", "return self")],
             ]
         handlers = add_ordered_mock_handlers(o, meth_spec)
-        o.add_handler(urllib2.UnknownHandler())
+        o.add_handler(urllib.request.UnknownHandler())
         for scheme in "do", "proxy", "redirect":
             self.assertRaises(URLError, o.open, scheme+"://example.com/")
 
         handlers = add_ordered_mock_handlers(o, meth_spec)
 
         req = Request("http://example.com/")
-        self.assertRaises(urllib2.URLError, o.open, req)
+        self.assertRaises(urllib.error.URLError, o.open, req)
         self.assertEqual(o.calls, [(handlers[0], "http_open", (req,), {})])
 
 ##     def test_error(self):
 
 
 def sanepathname2url(path):
-    import urllib
-    urlpath = urllib.pathname2url(path)
+    urlpath = urllib.request.pathname2url(path)
     if os.name == "nt" and urlpath.startswith("///"):
         urlpath = urlpath[2:]
     # XXX don't ask me about the mac...
                 self.filename, self.filetype = filename, filetype
                 return io.StringIO(self.data), len(self.data)
 
-        class NullFTPHandler(urllib2.FTPHandler):
+        class NullFTPHandler(urllib.request.FTPHandler):
             def __init__(self, data): self.data = data
             def connect_ftp(self, user, passwd, host, port, dirs,
                             timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
 
     def test_file(self):
         import email.utils, socket
-        h = urllib2.FileHandler()
+        h = urllib.request.FileHandler()
         o = h.parent = MockOpener()
 
         TESTFN = support.TESTFN
                 finally:
                     f.close()
 
-                self.assertRaises(urllib2.URLError,
+                self.assertRaises(urllib.error.URLError,
                                   h.file_open, Request(url))
             finally:
                 os.remove(TESTFN)
 
-        h = urllib2.FileHandler()
+        h = urllib.request.FileHandler()
         o = h.parent = MockOpener()
         # XXXX why does // mean ftp (and /// mean not ftp!), and where
         #  is file: scheme specified?  I think this is really a bug, and
             try:
                 h.file_open(req)
             # XXXX remove OSError when bug fixed
-            except (urllib2.URLError, OSError):
+            except (urllib.error.URLError, OSError):
                 self.assert_(not ftp)
             else:
                 self.assert_(o.req is req)
                 return ''
         class MockHTTPClass:
             def __init__(self):
+                self.level = 0
                 self.req_headers = []
                 self.data = None
                 self.raise_on_endheaders = False
             def getresponse(self):
                 return MockHTTPResponse(MockFile(), {}, 200, "OK")
 
-        h = urllib2.AbstractHTTPHandler()
+        h = urllib.request.AbstractHTTPHandler()
         o = h.parent = MockOpener()
 
         url = "http://example.com/"
 
         # check socket.error converted to URLError
         http.raise_on_endheaders = True
-        self.assertRaises(urllib2.URLError, h.do_open, http, req)
+        self.assertRaises(urllib.error.URLError, h.do_open, http, req)
 
         # check adding of standard headers
         o.addheaders = [("Spam", "eggs")]
             self.assertEqual(req.unredirected_hdrs["Spam"], "foo")
 
     def test_errors(self):
-        h = urllib2.HTTPErrorProcessor()
+        h = urllib.request.HTTPErrorProcessor()
         o = h.parent = MockOpener()
 
         url = "http://example.com/"
 
     def test_cookies(self):
         cj = MockCookieJar()
-        h = urllib2.HTTPCookieProcessor(cj)
+        h = urllib.request.HTTPCookieProcessor(cj)
         o = h.parent = MockOpener()
 
         req = Request("http://example.com/")
     def test_redirect(self):
         from_url = "http://example.com/a.html"
         to_url = "http://example.com/b.html"
-        h = urllib2.HTTPRedirectHandler()
+        h = urllib.request.HTTPRedirectHandler()
         o = h.parent = MockOpener()
 
         # ordinary redirect behaviour
                 try:
                     method(req, MockFile(), code, "Blah",
                            MockHeaders({"location": to_url}))
-                except urllib2.HTTPError:
+                except urllib.error.HTTPError:
                     # 307 in response to POST requires user OK
                     self.assert_(code == 307 and data is not None)
                 self.assertEqual(o.req.get_full_url(), to_url)
             while 1:
                 redirect(h, req, "http://example.com/")
                 count = count + 1
-        except urllib2.HTTPError:
+        except urllib.error.HTTPError:
             # don't stop until max_repeats, because cookies may introduce state
-            self.assertEqual(count, urllib2.HTTPRedirectHandler.max_repeats)
+            self.assertEqual(count, urllib.request.HTTPRedirectHandler.max_repeats)
 
         # detect endless non-repeating chain of redirects
         req = Request(from_url, origin_req_host="example.com")
             while 1:
                 redirect(h, req, "http://example.com/%d" % count)
                 count = count + 1
-        except urllib2.HTTPError:
+        except urllib.error.HTTPError:
             self.assertEqual(count,
-                             urllib2.HTTPRedirectHandler.max_redirections)
+                             urllib.request.HTTPRedirectHandler.max_redirections)
 
     def test_cookie_redirect(self):
         # cookies shouldn't leak into redirected requests
         cj = CookieJar()
         interact_netscape(cj, "http://www.example.com/", "spam=eggs")
         hh = MockHTTPHandler(302, "Location: http://www.cracker.com/\r\n\r\n")
-        hdeh = urllib2.HTTPDefaultErrorHandler()
-        hrh = urllib2.HTTPRedirectHandler()
-        cp = urllib2.HTTPCookieProcessor(cj)
+        hdeh = urllib.request.HTTPDefaultErrorHandler()
+        hrh = urllib.request.HTTPRedirectHandler()
+        cp = urllib.request.HTTPCookieProcessor(cj)
         o = build_test_opener(hh, hdeh, hrh, cp)
         o.open("http://www.example.com/")
         self.assert_(not hh.req.has_header("Cookie"))
 
     def test_proxy(self):
         o = OpenerDirector()
-        ph = urllib2.ProxyHandler(dict(http="proxy.example.com:3128"))
+        ph = urllib.request.ProxyHandler(dict(http="proxy.example.com:3128"))
         o.add_handler(ph)
         meth_spec = [
             [("http_open", "return response")]
     def test_basic_auth(self, quote_char='"'):
         opener = OpenerDirector()
         password_manager = MockPasswordManager()
-        auth_handler = urllib2.HTTPBasicAuthHandler(password_manager)
+        auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager)
         realm = "ACME Widget Store"
         http_handler = MockHTTPHandler(
             401, 'WWW-Authenticate: Basic realm=%s%s%s\r\n\r\n' %
 
     def test_proxy_basic_auth(self):
         opener = OpenerDirector()
-        ph = urllib2.ProxyHandler(dict(http="proxy.example.com:3128"))
+        ph = urllib.request.ProxyHandler(dict(http="proxy.example.com:3128"))
         opener.add_handler(ph)
         password_manager = MockPasswordManager()
-        auth_handler = urllib2.ProxyBasicAuthHandler(password_manager)
+        auth_handler = urllib.request.ProxyBasicAuthHandler(password_manager)
         realm = "ACME Networks"
         http_handler = MockHTTPHandler(
             407, 'Proxy-Authenticate: Basic realm="%s"\r\n\r\n' % realm)
                 self.recorded = []
             def record(self, info):
                 self.recorded.append(info)
-        class TestDigestAuthHandler(urllib2.HTTPDigestAuthHandler):
+        class TestDigestAuthHandler(urllib.request.HTTPDigestAuthHandler):
             def http_error_401(self, *args, **kwds):
                 self.parent.record("digest")
-                urllib2.HTTPDigestAuthHandler.http_error_401(self,
+                urllib.request.HTTPDigestAuthHandler.http_error_401(self,
                                                              *args, **kwds)
-        class TestBasicAuthHandler(urllib2.HTTPBasicAuthHandler):
+        class TestBasicAuthHandler(urllib.request.HTTPBasicAuthHandler):
             def http_error_401(self, *args, **kwds):
                 self.parent.record("basic")
-                urllib2.HTTPBasicAuthHandler.http_error_401(self,
+                urllib.request.HTTPBasicAuthHandler.http_error_401(self,
                                                             *args, **kwds)
 
         opener = RecordingOpenerDirector()
 class MiscTests(unittest.TestCase):
 
     def test_build_opener(self):
-        class MyHTTPHandler(urllib2.HTTPHandler): pass
-        class FooHandler(urllib2.BaseHandler):
+        class MyHTTPHandler(urllib.request.HTTPHandler): pass
+        class FooHandler(urllib.request.BaseHandler):
             def foo_open(self): pass
-        class BarHandler(urllib2.BaseHandler):
+        class BarHandler(urllib.request.BaseHandler):
             def bar_open(self): pass
 
-        build_opener = urllib2.build_opener
+        build_opener = urllib.request.build_opener
 
         o = build_opener(FooHandler, BarHandler)
         self.opener_has_handler(o, FooHandler)
         # a particular case of overriding: default handlers can be passed
         # in explicitly
         o = build_opener()
-        self.opener_has_handler(o, urllib2.HTTPHandler)
-        o = build_opener(urllib2.HTTPHandler)
-        self.opener_has_handler(o, urllib2.HTTPHandler)
-        o = build_opener(urllib2.HTTPHandler())
-        self.opener_has_handler(o, urllib2.HTTPHandler)
+        self.opener_has_handler(o, urllib.request.HTTPHandler)
+        o = build_opener(urllib.request.HTTPHandler)
+        self.opener_has_handler(o, urllib.request.HTTPHandler)
+        o = build_opener(urllib.request.HTTPHandler())
+        self.opener_has_handler(o, urllib.request.HTTPHandler)
 
         # Issue2670: multiple handlers sharing the same base class
-        class MyOtherHTTPHandler(urllib2.HTTPHandler): pass
+        class MyOtherHTTPHandler(urllib.request.HTTPHandler): pass
         o = build_opener(MyHTTPHandler, MyOtherHTTPHandler)
         self.opener_has_handler(o, MyHTTPHandler)
         self.opener_has_handler(o, MyOtherHTTPHandler)
 def test_main(verbose=None):
     from test import test_urllib2
     support.run_doctest(test_urllib2, verbose)
-    support.run_doctest(urllib2, verbose)
+    support.run_doctest(urllib.request, verbose)
     tests = (TrivialTests,
              OpenerDirectorTests,
              HandlerTests,

Lib/test/test_urllib2_localnet.py

 
 import email
 import threading
-import urlparse
-import urllib2
+import urllib.parse
+import urllib.request
 import http.server
 import unittest
 import hashlib
         self._stop_server = False
         self.ready = threading.Event()
         request_handler.protocol_version = "HTTP/1.0"
-        self.httpd = LoopbackHttpServer(('127.0.0.1', 0),
+        self.httpd = LoopbackHttpServer(("127.0.0.1", 0),
                                         request_handler)
         #print "Serving HTTP on %s port %s" % (self.httpd.server_name,
         #                                      self.httpd.server_port)
         if len(self._users) == 0:
             return True
 
-        if 'Proxy-Authorization' not in request_handler.headers:
+        if "Proxy-Authorization" not in request_handler.headers:
             return self._return_auth_challenge(request_handler)
         else:
             auth_dict = self._create_auth_dict(
-                request_handler.headers['Proxy-Authorization']
+                request_handler.headers["Proxy-Authorization"]
                 )
             if auth_dict["username"] in self._users:
                 password = self._users[ auth_dict["username"] ]
 
     def log_message(self, format, *args):
         # Uncomment the next line for debugging.
-        #sys.stderr.write(format % args)
+        # sys.stderr.write(format % args)
         pass
 
     def do_GET(self):
-        (scm, netloc, path, params, query, fragment) = urlparse.urlparse(
-            self.path, 'http')
+        (scm, netloc, path, params, query, fragment) = urllib.parse.urlparse(
+            self.path, "http")
         self.short_path = path
         if self.digest_auth_handler.handle_request(self):
             self.send_response(200, "OK")
         self.server.start()
         self.server.ready.wait()
         proxy_url = "http://127.0.0.1:%d" % self.server.port
-        handler = urllib2.ProxyHandler({"http" : proxy_url})
-        self._digest_auth_handler = urllib2.ProxyDigestAuthHandler()
-        self.opener = urllib2.build_opener(handler, self._digest_auth_handler)
+        handler = urllib.request.ProxyHandler({"http" : proxy_url})
+        self._digest_auth_handler = urllib.request.ProxyDigestAuthHandler()
+        self.opener = urllib.request.build_opener(
+            handler, self._digest_auth_handler)
 
     def tearDown(self):
         self.server.stop()
         self._digest_auth_handler.add_password(self.REALM, self.URL,
                                                self.USER, self.PASSWD+"bad")
         FakeProxyHandler.digest_auth_handler.set_qop("auth")
-        self.assertRaises(urllib2.HTTPError,
+        self.assertRaises(urllib.error.HTTPError,
                           self.opener.open,
                           self.URL)
 
     def test_proxy_with_no_password_raises_httperror(self):
         FakeProxyHandler.digest_auth_handler.set_qop("auth")
-        self.assertRaises(urllib2.HTTPError,
+        self.assertRaises(urllib.error.HTTPError,
                           self.opener.open,
                           self.URL)
 
         FakeProxyHandler.digest_auth_handler.set_qop("auth-int")
         try:
             result = self.opener.open(self.URL)
-        except urllib2.URLError:
+        except urllib.error.URLError:
             # It's okay if we don't support auth-int, but we certainly
             # shouldn't receive any kind of exception here other than
             # a URLError.
                 self.wfile.write(body)
 
         def do_POST(self):
-            content_length = self.headers['Content-Length']
+            content_length = self.headers["Content-Length"]
             post_data = self.rfile.read(int(content_length))
             self.do_GET()
             self.requests.append(post_data)
             for (header, value) in headers:
                 self.send_header(header, value % self.port)
             if body:
-                self.send_header('Content-type', 'text/plain')
+                self.send_header("Content-type", "text/plain")
                 self.end_headers()
                 return body
             self.end_headers()
     for transparent redirection have been written.
     """
 
-    def start_server(self, responses):
+    def setUp(self):
+        self.server = None
+
+    def tearDown(self):
+        if self.server is not None:
+            self.server.stop()
+
+    def urlopen(self, url, data=None):
+        f = urllib.request.urlopen(url, data)
+        result = f.read()
+        f.close()
+        return result
+
+    def start_server(self, responses=None):
+        if responses is None:
+            responses = [(200, [], b"we don't care")]
         handler = GetRequestHandler(responses)
 
         self.server = LoopbackHttpServerThread(handler)
         handler.port = port
         return handler
 
-
     def test_redirection(self):
-        expected_response = b'We got here...'
+        expected_response = b"We got here..."
         responses = [
-            (302, [('Location', 'http://localhost:%s/somewhere_else')], ''),
+            (302, [("Location", "http://localhost:%s/somewhere_else")], ""),
             (200, [], expected_response)
         ]
 
         handler = self.start_server(responses)
-
-        try:
-            f = urllib2.urlopen('http://localhost:%s/' % handler.port)
-            data = f.read()
-            f.close()
-
-            self.assertEquals(data, expected_response)
-            self.assertEquals(handler.requests, ['/', '/somewhere_else'])
-        finally:
-            self.server.stop()
-
+        data = self.urlopen("http://localhost:%s/" % handler.port)
+        self.assertEquals(data, expected_response)
+        self.assertEquals(handler.requests, ["/", "/somewhere_else"])
 
     def test_404(self):
-        expected_response = b'Bad bad bad...'
+        expected_response = b"Bad bad bad..."
         handler = self.start_server([(404, [], expected_response)])
 
         try:
-            try:
-                urllib2.urlopen('http://localhost:%s/weeble' % handler.port)
-            except urllib2.URLError as f:
-                data = f.read()
-                f.close()
-            else:
-                self.fail('404 should raise URLError')
+            self.urlopen("http://localhost:%s/weeble" % handler.port)
+        except urllib.error.URLError as f:
+            data = f.read()
+            f.close()
+        else:
+            self.fail("404 should raise URLError")
 
-            self.assertEquals(data, expected_response)
-            self.assertEquals(handler.requests, ['/weeble'])
-        finally:
-            self.server.stop()
-
+        self.assertEquals(data, expected_response)
+        self.assertEquals(handler.requests, ["/weeble"])
 
     def test_200(self):
-        expected_response = b'pycon 2008...'
+        expected_response = b"pycon 2008..."
         handler = self.start_server([(200, [], expected_response)])
-
-        try:
-            f = urllib2.urlopen('http://localhost:%s/bizarre' % handler.port)
-            data = f.read()
-            f.close()
-
-            self.assertEquals(data, expected_response)
-            self.assertEquals(handler.requests, ['/bizarre'])
-        finally:
-            self.server.stop()
+        data = self.urlopen("http://localhost:%s/bizarre" % handler.port)
+        self.assertEquals(data, expected_response)
+        self.assertEquals(handler.requests, ["/bizarre"])
 
     def test_200_with_parameters(self):
-        expected_response = b'pycon 2008...'
+        expected_response = b"pycon 2008..."
         handler = self.start_server([(200, [], expected_response)])
-
-        try:
-            f = urllib2.urlopen('http://localhost:%s/bizarre' % handler.port, b'get=with_feeling')
-            data = f.read()
-            f.close()
-
-            self.assertEquals(data, expected_response)
-            self.assertEquals(handler.requests, ['/bizarre', b'get=with_feeling'])
-        finally:
-            self.server.stop()
-
+        data = self.urlopen("http://localhost:%s/bizarre" % handler.port,
+                             b"get=with_feeling")
+        self.assertEquals(data, expected_response)
+        self.assertEquals(handler.requests, ["/bizarre", b"get=with_feeling"])
 
     def test_sending_headers(self):
-        handler = self.start_server([(200, [], b"we don't care")])
-
-        try:
-            req = urllib2.Request("http://localhost:%s/" % handler.port,
-                                  headers={'Range': 'bytes=20-39'})
-            urllib2.urlopen(req)
-            self.assertEqual(handler.headers_received['Range'], 'bytes=20-39')
-        finally:
-            self.server.stop()
+        handler = self.start_server()
+        req = urllib.request.Request("http://localhost:%s/" % handler.port,
+                                     headers={"Range": "bytes=20-39"})
+        urllib.request.urlopen(req)
+        self.assertEqual(handler.headers_received["Range"], "bytes=20-39")
 
     def test_basic(self):
-        handler = self.start_server([(200, [], b"we don't care")])
-
+        handler = self.start_server()
+        open_url = urllib.request.urlopen("http://localhost:%s" % handler.port)
+        for attr in ("read", "close", "info", "geturl"):
+            self.assert_(hasattr(open_url, attr), "object returned from "
+                         "urlopen lacks the %s attribute" % attr)
         try:
-            open_url = urllib2.urlopen("http://localhost:%s" % handler.port)
-            for attr in ("read", "close", "info", "geturl"):