Deleted ancient file.
This commit is contained in:
Guido van Rossum
parent
b655939d69
commit
94bf62f9c0
410
Lib/urlopen.py
410
Lib/urlopen.py
|
|
@ -1,410 +0,0 @@
|
|||
# Open an arbitrary URL
|
||||
#
|
||||
# See the following document for a tentative description of URLs:
|
||||
# Uniform Resource Locators Tim Berners-Lee
|
||||
# INTERNET DRAFT CERN
|
||||
# IETF URL Working Group 14 July 1993
|
||||
# draft-ietf-uri-url-01.txt
|
||||
#
|
||||
# The object returned by URLopener().open(file) will differ per
|
||||
# protocol. All you know is that is has methods read(), readline(),
|
||||
# readlines(), fileno(), close() and info(). The read*(), fileno()
|
||||
# and close() methods work like those of open files.
|
||||
# The info() method returns an rfc822.Message object which can be
|
||||
# used to query various info about the object, if available.
|
||||
# (rfc822.Message objects are queried with the getheader() method.)
|
||||
|
||||
import socket
|
||||
import regex
|
||||
|
||||
|
||||
# This really consists of two pieces:
|
||||
# (1) a class which handles opening of all sorts of URLs
|
||||
# (plus assorted utilities etc.)
|
||||
# (2) a set of functions for parsing URLs
|
||||
# XXX Should these be separated out into different modules?
|
||||
|
||||
|
||||
# Shortcut for basic usage
|
||||
_urlopener = None
|
||||
def urlopen(url):
|
||||
global _urlopener
|
||||
if not _urlopener:
|
||||
_urlopener = URLopener()
|
||||
return _urlopener.open(url)
|
||||
def urlretrieve(url):
|
||||
global _urlopener
|
||||
if not _urlopener:
|
||||
_urlopener = URLopener()
|
||||
return _urlopener.retrieve(url)
|
||||
def urlcleanup():
|
||||
if _urlopener:
|
||||
_urlopener.cleanup()
|
||||
|
||||
|
||||
# Class to open URLs.
|
||||
# This is a class rather than just a subroutine because we may need
|
||||
# more than one set of global protocol-specific options.
|
||||
ftpcache = {}
|
||||
class URLopener:
|
||||
|
||||
# Constructor
|
||||
def __init__(self):
|
||||
self.addheaders = []
|
||||
self.tempcache = {}
|
||||
self.ftpcache = ftpcache
|
||||
# Undocumented feature: you can use a different
|
||||
# ftp cache by assigning to the .ftpcache member;
|
||||
# in case you want logically independent URL openers
|
||||
|
||||
def __del__(self):
|
||||
self.close()
|
||||
|
||||
def close(self):
|
||||
self.cleanup()
|
||||
|
||||
def cleanup(self):
|
||||
import os
|
||||
for url in self.tempcache.keys():
|
||||
try:
|
||||
os.unlink(self.tempcache[url][0])
|
||||
except os.error:
|
||||
pass
|
||||
del self.tempcache[url]
|
||||
|
||||
# Add a header to be used by the HTTP interface only
|
||||
# e.g. u.addheader('Accept', 'sound/basic')
|
||||
def addheader(self, *args):
|
||||
self.addheaders.append(args)
|
||||
|
||||
# External interface
|
||||
# Use URLopener().open(file) instead of open(file, 'r')
|
||||
def open(self, url):
|
||||
type, url = splittype(unwrap(url))
|
||||
if not type: type = 'file'
|
||||
name = 'open_' + type
|
||||
if '-' in name:
|
||||
import regsub
|
||||
name = regsub.gsub('-', '_', name)
|
||||
if not hasattr(self, name):
|
||||
raise IOError, ('url error', 'unknown url type', type)
|
||||
try:
|
||||
return getattr(self, name)(url)
|
||||
except socket.error, msg:
|
||||
raise IOError, ('socket error', msg)
|
||||
|
||||
# External interface
|
||||
# retrieve(url) returns (filename, None) for a local object
|
||||
# or (tempfilename, headers) for a remote object
|
||||
def retrieve(self, url):
|
||||
if self.tempcache.has_key(url):
|
||||
return self.tempcache[url]
|
||||
url1 = unwrap(url)
|
||||
if self.tempcache.has_key(url1):
|
||||
self.tempcache[url] = self.tempcache[url1]
|
||||
return self.tempcache[url1]
|
||||
type, url1 = splittype(url1)
|
||||
if not type or type == 'file':
|
||||
try:
|
||||
fp = self.open_local_file(url1)
|
||||
del fp
|
||||
return splithost(url1)[1], None
|
||||
except IOError, msg:
|
||||
pass
|
||||
fp = self.open(url)
|
||||
headers = fp.info()
|
||||
import tempfile
|
||||
tfn = tempfile.mktemp()
|
||||
self.tempcache[url] = result = tfn, headers
|
||||
tfp = open(tfn, 'w')
|
||||
bs = 1024*8
|
||||
block = fp.read(bs)
|
||||
while block:
|
||||
tfp.write(block)
|
||||
block = fp.read(bs)
|
||||
del fp
|
||||
del tfp
|
||||
return result
|
||||
|
||||
# Each method named open_<type> knows how to open that type of URL
|
||||
|
||||
# Use HTTP protocol
|
||||
def open_http(self, url):
|
||||
import httplib
|
||||
host, selector = splithost(url)
|
||||
h = httplib.HTTP(host)
|
||||
h.putrequest('GET', selector)
|
||||
for args in self.addheaders: apply(h.putheader, args)
|
||||
errcode, errmsg, headers = h.getreply()
|
||||
if errcode == 200: return addinfo(h.getfile(), headers)
|
||||
else: raise IOError, ('http error', errcode, errmsg, headers)
|
||||
|
||||
# Use Gopher protocol
|
||||
def open_gopher(self, url):
|
||||
import gopherlib
|
||||
host, selector = splithost(url)
|
||||
type, selector = splitgophertype(selector)
|
||||
selector, query = splitquery(selector)
|
||||
if query: fp = gopherlib.send_query(selector, query, host)
|
||||
else: fp = gopherlib.send_selector(selector, host)
|
||||
return addinfo(fp, noheaders())
|
||||
|
||||
# Use local file or FTP depending on form of URL
|
||||
def open_file(self, url):
|
||||
try:
|
||||
return self.open_local_file(url)
|
||||
except IOError:
|
||||
return self.open_ftp(url)
|
||||
|
||||
# Use local file
|
||||
def open_local_file(self, url):
|
||||
host, file = splithost(url)
|
||||
if not host: return addinfo(open(file, 'r'), noheaders())
|
||||
host, port = splitport(host)
|
||||
if not port and socket.gethostbyname(host) in (
|
||||
localhost(), thishost()):
|
||||
return addinfo(open(file, 'r'), noheaders())
|
||||
raise IOError, ('local file error', 'not on local host')
|
||||
|
||||
# Use FTP protocol
|
||||
def open_ftp(self, url):
|
||||
host, file = splithost(url)
|
||||
if not host: raise IOError, ('ftp error', 'no host given')
|
||||
host, port = splitport(host)
|
||||
host = socket.gethostbyname(host)
|
||||
if not port:
|
||||
import ftplib
|
||||
port = ftplib.FTP_PORT
|
||||
key = (host, port)
|
||||
try:
|
||||
if not self.ftpcache.has_key(key):
|
||||
self.ftpcache[key] = ftpwrapper(host, port)
|
||||
return addinfo(self.ftpcache[key].retrfile(file),
|
||||
noheaders())
|
||||
except ftperrors(), msg:
|
||||
raise IOError, ('ftp error', msg)
|
||||
|
||||
|
||||
# Utility functions
|
||||
|
||||
# Return the IP address of the magic hostname 'localhost'
|
||||
_localhost = None
|
||||
def localhost():
|
||||
global _localhost
|
||||
if not _localhost:
|
||||
_localhost = socket.gethostbyname('localhost')
|
||||
return _localhost
|
||||
|
||||
# Return the IP address of the current host
|
||||
_thishost = None
|
||||
def thishost():
|
||||
global _thishost
|
||||
if not _thishost:
|
||||
_thishost = socket.gethostbyname(socket.gethostname())
|
||||
return _thishost
|
||||
|
||||
# Return the set of errors raised by the FTP class
|
||||
_ftperrors = None
|
||||
def ftperrors():
|
||||
global _ftperrors
|
||||
if not _ftperrors:
|
||||
import ftplib
|
||||
_ftperrors = (ftplib.error_reply,
|
||||
ftplib.error_temp,
|
||||
ftplib.error_perm,
|
||||
ftplib.error_proto)
|
||||
return _ftperrors
|
||||
|
||||
# Return an empty rfc822.Message object
|
||||
_noheaders = None
|
||||
def noheaders():
|
||||
global _noheaders
|
||||
if not _noheaders:
|
||||
import rfc822
|
||||
_noheaders = rfc822.Message(open('/dev/null', 'r'))
|
||||
_noheaders.fp.close() # Recycle file descriptor
|
||||
return _noheaders
|
||||
|
||||
|
||||
# Utility classes
|
||||
|
||||
# Class used by open_ftp() for cache of open FTP connections
|
||||
class ftpwrapper:
|
||||
def __init__(self, host, port):
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.init()
|
||||
def init(self):
|
||||
import ftplib
|
||||
self.ftp = ftplib.FTP()
|
||||
self.ftp.connect(self.host, self.port)
|
||||
self.ftp.login()
|
||||
def retrfile(self, file):
|
||||
import ftplib
|
||||
try:
|
||||
self.ftp.voidcmd('TYPE I')
|
||||
except ftplib.all_errors:
|
||||
self.init()
|
||||
self.ftp.voidcmd('TYPE I')
|
||||
conn = None
|
||||
if file:
|
||||
try:
|
||||
cmd = 'RETR ' + file
|
||||
conn = self.ftp.transfercmd(cmd)
|
||||
except ftplib.error_perm, reason:
|
||||
if reason[:3] != '550':
|
||||
raise IOError, ('ftp error', reason)
|
||||
if not conn:
|
||||
# Try a directory listing
|
||||
if file: cmd = 'LIST ' + file
|
||||
else: cmd = 'LIST'
|
||||
conn = self.ftp.transfercmd(cmd)
|
||||
return addclosehook(conn.makefile('r'), self.ftp.voidresp)
|
||||
|
||||
# Base class for addinfo and addclosehook
|
||||
class addbase:
|
||||
def __init__(self, fp):
|
||||
self.fp = fp
|
||||
self.read = self.fp.read
|
||||
self.readline = self.fp.readline
|
||||
self.readlines = self.fp.readlines
|
||||
self.fileno = self.fp.fileno
|
||||
def __repr__(self):
|
||||
return '<%s at %s whose fp = %s>' % (
|
||||
self.__class__.__name__, `id(self)`, `self.fp`)
|
||||
def __del__(self):
|
||||
self.close()
|
||||
def close(self):
|
||||
self.read = None
|
||||
self.readline = None
|
||||
self.readlines = None
|
||||
self.fileno = None
|
||||
self.fp = None
|
||||
|
||||
# Class to add a close hook to an open file
|
||||
class addclosehook(addbase):
|
||||
def __init__(self, fp, closehook, *hookargs):
|
||||
addbase.__init__(self, fp)
|
||||
self.closehook = closehook
|
||||
self.hookargs = hookargs
|
||||
def close(self):
|
||||
if self.closehook:
|
||||
apply(self.closehook, self.hookargs)
|
||||
self.closehook = None
|
||||
self.hookargs = None
|
||||
addbase.close(self)
|
||||
|
||||
# class to add an info() method to an open file
|
||||
class addinfo(addbase):
|
||||
def __init__(self, fp, headers):
|
||||
addbase.__init__(self, fp)
|
||||
self.headers = headers
|
||||
def info(self):
|
||||
return self.headers
|
||||
|
||||
|
||||
# Utility to combine a URL with a base URL to form a new URL
|
||||
|
||||
def basejoin(base, url):
|
||||
type, path = splittype(url)
|
||||
if type: return url
|
||||
host, path = splithost(path)
|
||||
basetype, basepath = splittype(base)
|
||||
basehost, basepath = splithost(basepath)
|
||||
basepath, basetag = splittag(basepath)
|
||||
basepath, basequery = splitquery(basepath)
|
||||
type = basetype or 'file'
|
||||
if path[:1] != '/':
|
||||
import string
|
||||
i = string.rfind(basepath, '/')
|
||||
if i < 0: basepath = '/'
|
||||
else: basepath = basepath[:i+1]
|
||||
path = basepath + path
|
||||
if not host: host = basehost
|
||||
if host: return type + '://' + host + path
|
||||
else: return type + ':' + path
|
||||
|
||||
|
||||
# Utilities to parse URLs:
|
||||
# unwrap('<URL:type//host/path>') --> 'type//host/path'
|
||||
# splittype('type:opaquestring') --> 'type', 'opaquestring'
|
||||
# splithost('//host[:port]/path') --> 'host[:port]', '/path'
|
||||
# splitport('host:port') --> 'host', 'port'
|
||||
# splitquery('/path?query') --> '/path', 'query'
|
||||
# splittag('/path#tag') --> '/path', 'tag'
|
||||
# splitgophertype('/Xselector') --> 'X', 'selector'
|
||||
|
||||
def unwrap(url):
|
||||
import string
|
||||
url = string.strip(url)
|
||||
if url[:1] == '<' and url[-1:] == '>':
|
||||
url = string.strip(url[1:-1])
|
||||
if url[:4] == 'URL:': url = string.strip(url[4:])
|
||||
return url
|
||||
|
||||
_typeprog = regex.compile('^\([^/:]+\):\(.*\)$')
|
||||
def splittype(url):
|
||||
if _typeprog.match(url) >= 0: return _typeprog.group(1, 2)
|
||||
return None, url
|
||||
|
||||
_hostprog = regex.compile('^//\([^/]+\)\(.*\)$')
|
||||
def splithost(url):
|
||||
if _hostprog.match(url) >= 0: return _hostprog.group(1, 2)
|
||||
return None, url
|
||||
|
||||
_portprog = regex.compile('^\(.*\):\([0-9]+\)$')
|
||||
def splitport(host):
|
||||
if _portprog.match(host) >= 0: return _portprog.group(1, 2)
|
||||
return host, None
|
||||
|
||||
_queryprog = regex.compile('^\(.*\)\?\([^?]*\)$')
|
||||
def splitquery(url):
|
||||
if _queryprog.match(url) >= 0: return _queryprog.group(1, 2)
|
||||
return url, None
|
||||
|
||||
_tagprog = regex.compile('^\(.*\)#\([^#]*\)$')
|
||||
def splittag(url):
|
||||
if _tagprog.match(url) >= 0: return _tagprog.group(1, 2)
|
||||
return url, None
|
||||
|
||||
def splitgophertype(selector):
|
||||
if selector[:1] == '/' and selector[1:2]:
|
||||
return selector[1], selector[2:]
|
||||
return None, selector
|
||||
|
||||
|
||||
# Test program
|
||||
def test():
|
||||
import sys
|
||||
import regsub
|
||||
args = sys.argv[1:]
|
||||
if not args:
|
||||
args = [
|
||||
'/etc/passwd',
|
||||
'file:/etc/passwd',
|
||||
'file://localhost/etc/passwd',
|
||||
'ftp://ftp.cwi.nl/etc/passwd',
|
||||
'gopher://gopher.cwi.nl/11/',
|
||||
'http://www.cwi.nl/index.html',
|
||||
]
|
||||
try:
|
||||
for url in args:
|
||||
print '-'*10, url, '-'*10
|
||||
fn, h = urlretrieve(url)
|
||||
print fn, h
|
||||
if h:
|
||||
print '======'
|
||||
for k in h.keys(): print k + ':', h[k]
|
||||
print '======'
|
||||
fp = open(fn, 'r')
|
||||
data = fp.read()
|
||||
del fp
|
||||
print regsub.gsub('\r', '', data)
|
||||
fn, h = None, None
|
||||
print '-'*40
|
||||
finally:
|
||||
urlcleanup()
|
||||
|
||||
# Run test program when run as a script
|
||||
if __name__ == '__main__':
|
||||
test()
|
||||
Loading…
Reference in New Issue