mirror of https://github.com/python/cpython
Towards a standard access mechanism for URLs.
This commit is contained in:
Guido van Rossum
parent
3da56c3686
commit
23acc9590c
|
|
@ -0,0 +1,195 @@
|
|||
# Gopher protocol client interface
|
||||
|
||||
import string
|
||||
|
||||
# Default selector, host and port
|
||||
DEF_SELECTOR = '1/'
|
||||
DEF_HOST = 'gopher.micro.umn.edu'
|
||||
DEF_PORT = 70
|
||||
|
||||
# Recognized file types
|
||||
A_TEXT = '0'
|
||||
A_MENU = '1'
|
||||
A_CSO = '2'
|
||||
A_ERROR = '3'
|
||||
A_MACBINHEX = '4'
|
||||
A_PCBINHEX = '5'
|
||||
A_UUENCODED = '6'
|
||||
A_INDEX = '7'
|
||||
A_TELNET = '8'
|
||||
A_BINARY = '9'
|
||||
A_DUPLICATE = '+'
|
||||
A_SOUND = 's'
|
||||
A_EVENT = 'e'
|
||||
A_CALENDAR = 'c'
|
||||
A_HTML = 'h'
|
||||
A_TN3270 = 'T'
|
||||
A_MIME = 'M'
|
||||
A_IMAGE = 'I'
|
||||
A_WHOIS = 'w'
|
||||
A_QUERY = 'q'
|
||||
A_GIF = 'g'
|
||||
A_HTML = 'h' # HTML file
|
||||
A_WWW = 'w' # WWW address
|
||||
A_PLUS_IMAGE = ':'
|
||||
A_PLUS_MOVIE = ';'
|
||||
A_PLUS_SOUND = '<'
|
||||
|
||||
|
||||
# Function mapping all file types to strings; unknown types become TYPE='x'
|
||||
_names = dir()
|
||||
_type_to_name_map = None
|
||||
def type_to_name(gtype):
|
||||
global _type_to_name_map
|
||||
if not _type_to_name_map:
|
||||
for name in _names:
|
||||
if name[:2] == 'A_':
|
||||
_type_to_name_map[eval(name)] = name[2:]
|
||||
if _type_to_name_map.has_key(gtype):
|
||||
return _type_to_name_map[gtype]
|
||||
return 'TYPE=' + `gtype`
|
||||
|
||||
# Names for characters and strings
|
||||
CRLF = '\r\n'
|
||||
TAB = '\t'
|
||||
|
||||
# Send a selector to a given host and port, return a file with the reply
|
||||
def send_selector(selector, host, *args):
|
||||
import socket
|
||||
import string
|
||||
if args:
|
||||
if args[1:]: raise TypeError, 'too many args'
|
||||
port = args[0]
|
||||
else:
|
||||
port = None
|
||||
i = string.find(host, ':')
|
||||
if i >= 0:
|
||||
host, port = host[:i], string.atoi(host[i+1:])
|
||||
if not port:
|
||||
port = DEF_PORT
|
||||
elif type(port) == type(''):
|
||||
port = string.atoi(port)
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
s.connect(host, port)
|
||||
s.send(selector + CRLF)
|
||||
s.shutdown(1)
|
||||
return s.makefile('r')
|
||||
|
||||
# Send a selector and a query string
|
||||
def send_query(selector, query, host, *args):
|
||||
return apply(send_selector, (selector + '\t' + query, host) + args)
|
||||
|
||||
# The following functions interpret the data returned by the gopher
|
||||
# server according to the expected type, e.g. textfile or directory
|
||||
|
||||
# Get a directory in the form of a list of entries
|
||||
def get_directory(f):
|
||||
import string
|
||||
list = []
|
||||
while 1:
|
||||
line = f.readline()
|
||||
if not line:
|
||||
print '(Unexpected EOF from server)'
|
||||
break
|
||||
if line[-2:] == CRLF:
|
||||
line = line[:-2]
|
||||
elif line[-1:] in CRLF:
|
||||
line = line[:-1]
|
||||
if line == '.':
|
||||
break
|
||||
if not line:
|
||||
print '(Empty line from server)'
|
||||
continue
|
||||
gtype = line[0]
|
||||
parts = string.splitfields(line[1:], TAB)
|
||||
if len(parts) < 4:
|
||||
print '(Bad line from server:', `line`, ')'
|
||||
continue
|
||||
if len(parts) > 4:
|
||||
if parts[4:] != ['+']:
|
||||
print '(Extra info from server:', parts[4:], ')'
|
||||
else:
|
||||
parts.append('')
|
||||
parts.insert(0, gtype)
|
||||
list.append(parts)
|
||||
return list
|
||||
|
||||
# Get a text file as a list of lines, with trailing CRLF stripped
|
||||
def get_textfile(f):
|
||||
list = []
|
||||
get_alt_textfile(f, list.append)
|
||||
return list
|
||||
|
||||
# Get a text file and pass each line to a function, with trailing CRLF stripped
|
||||
def get_alt_textfile(f, func):
|
||||
while 1:
|
||||
line = f.readline()
|
||||
if not line:
|
||||
print '(Unexpected EOF from server)'
|
||||
break
|
||||
if line[-2:] == CRLF:
|
||||
line = line[:-2]
|
||||
elif line[-1:] in CRLF:
|
||||
line = line[:-1]
|
||||
if line == '.':
|
||||
break
|
||||
if line[:2] == '..':
|
||||
line = line[1:]
|
||||
func(line)
|
||||
|
||||
# Get a binary file as one solid data block
|
||||
def get_binary(f):
|
||||
data = f.read()
|
||||
return data
|
||||
|
||||
# Get a binary file and pass each block to a function
|
||||
def get_alt_binary(f, func, blocksize):
|
||||
while 1:
|
||||
data = f.read(blocksize)
|
||||
if not data:
|
||||
break
|
||||
func(data)
|
||||
|
||||
# Trivial test program
|
||||
def test():
|
||||
import sys
|
||||
import getopt
|
||||
opts, args = getopt.getopt(sys.argv[1:], '')
|
||||
selector = DEF_SELECTOR
|
||||
type = selector[0]
|
||||
host = DEF_HOST
|
||||
port = DEF_PORT
|
||||
if args:
|
||||
host = args[0]
|
||||
args = args[1:]
|
||||
if args:
|
||||
type = args[0]
|
||||
args = args[1:]
|
||||
if len(type) > 1:
|
||||
type, selector = type[0], type
|
||||
else:
|
||||
selector = ''
|
||||
if args:
|
||||
selector = args[0]
|
||||
args = args[1:]
|
||||
query = ''
|
||||
if args:
|
||||
query = args[0]
|
||||
args = args[1:]
|
||||
if type == A_INDEX:
|
||||
f = send_query(selector, query, host)
|
||||
else:
|
||||
f = send_selector(selector, host)
|
||||
if type == A_TEXT:
|
||||
list = get_textfile(f)
|
||||
for item in list: print item
|
||||
elif type in (A_MENU, A_INDEX):
|
||||
list = get_directory(f)
|
||||
for item in list: print item
|
||||
else:
|
||||
data = get_binary(f)
|
||||
print 'binary data:', len(data), 'bytes:', `data[:100]`[:40]
|
||||
|
||||
# Run the test when run as script
|
||||
if __name__ == '__main__':
|
||||
test()
|
||||
|
|
@ -0,0 +1,129 @@
|
|||
# HTTP client class
|
||||
#
|
||||
# See the following document for a tentative protocol description:
|
||||
# Hypertext Transfer Protocol (HTTP) Tim Berners-Lee, CERN
|
||||
# Internet Draft 5 Nov 1993
|
||||
# draft-ietf-iiir-http-00.txt Expires 5 May 1994
|
||||
#
|
||||
# Example:
|
||||
#
|
||||
# >>> from httplib import HTTP
|
||||
# >>> h = HTTP('www.cwi.nl')
|
||||
# >>> h.putreqest('GET', '/index.html')
|
||||
# >>> h.putheader('Accept', 'text/html')
|
||||
# >>> h.putheader('Accept', 'text/plain')
|
||||
# >>> errcode, errmsg, headers = h.getreply()
|
||||
# >>> if errcode == 200:
|
||||
# ... f = h.getfile()
|
||||
# ... print f.read() # Print the raw HTML
|
||||
# ...
|
||||
# <TITLE>Home Page of CWI, Amsterdam</TITLE>
|
||||
# [...many more lines...]
|
||||
# >>>
|
||||
#
|
||||
# Note that an HTTP object is used for a single request -- to issue a
|
||||
# second request to the same server, you create a new HTTP object.
|
||||
# (This is in accordance with the protocol, which uses a new TCP
|
||||
# connection for each request.)
|
||||
|
||||
|
||||
import os
|
||||
import socket
|
||||
import string
|
||||
import regex
|
||||
import regsub
|
||||
import rfc822
|
||||
|
||||
HTTP_VERSION = 'HTTP/1.0'
|
||||
HTTP_PORT = 80
|
||||
|
||||
replypat = regsub.gsub('\\.', '\\\\.', HTTP_VERSION) + \
|
||||
'[ \t]+\([0-9][0-9][0-9]\)\(.*\)'
|
||||
replyprog = regex.compile(replypat)
|
||||
|
||||
class HTTP:
|
||||
|
||||
def __init__(self, *args):
|
||||
self.debuglevel = 0
|
||||
if args: apply(self.connect, args)
|
||||
|
||||
def set_debuglevel(self, debuglevel):
|
||||
self.debuglevel = debuglevel
|
||||
|
||||
def connect(self, host, *args):
|
||||
if args:
|
||||
if args[1:]: raise TypeError, 'too many args'
|
||||
port = args[0]
|
||||
else:
|
||||
i = string.find(host, ':')
|
||||
if i >= 0:
|
||||
host, port = host[:i], host[i+1:]
|
||||
try: port = string.atoi(port)
|
||||
except string.atoi_error: port = None
|
||||
if not port: port = HTTP_PORT
|
||||
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
if self.debuglevel > 0: print 'connect:', (host, port)
|
||||
self.sock.connect(host, port)
|
||||
|
||||
def send(self, str):
|
||||
if self.debuglevel > 0: print 'send:', `str`
|
||||
self.sock.send(str)
|
||||
|
||||
def putrequest(self, request, selector):
|
||||
str = '%s %s %s\r\n' % (request, selector, HTTP_VERSION)
|
||||
self.send(str)
|
||||
|
||||
def putheader(self, header, *args):
|
||||
str = '%s: %s\r\n' % (header, string.joinfields(args,'\r\n\t'))
|
||||
self.send(str)
|
||||
|
||||
def endheaders(self):
|
||||
self.send('\r\n')
|
||||
|
||||
def endrequest(self):
|
||||
if self.debuglevel > 0: print 'shutdown: 1'
|
||||
self.sock.shutdown(1)
|
||||
|
||||
def getreply(self):
|
||||
self.endrequest()
|
||||
self.file = self.sock.makefile('r')
|
||||
line = self.file.readline()
|
||||
if self.debuglevel > 0: print 'reply:', `line`
|
||||
if replyprog.match(line) < 0:
|
||||
self.headers = None
|
||||
return -1, line, self.headers
|
||||
errcode, errmsg = replyprog.group(1, 2)
|
||||
errcode = string.atoi(errcode)
|
||||
errmsg = string.strip(errmsg)
|
||||
self.headers = rfc822.Message(self.file)
|
||||
return errcode, errmsg, self.headers
|
||||
|
||||
def getfile(self):
|
||||
return self.file
|
||||
|
||||
|
||||
def test():
|
||||
import sys
|
||||
import getopt
|
||||
opts, args = getopt.getopt(sys.argv[1:], 'd')
|
||||
dl = 0
|
||||
for o, a in opts:
|
||||
if o == '-d': dl = dl + 1
|
||||
host = 'www.cwi.nl:80'
|
||||
selector = '/index.html'
|
||||
if args[0:]: host = args[0]
|
||||
if args[1:]: selector = args[1]
|
||||
h = HTTP()
|
||||
h.set_debuglevel(dl)
|
||||
h.connect(host)
|
||||
h.putrequest('GET', selector)
|
||||
errcode, errmsg, headers = h.getreply()
|
||||
print 'errcode =', errcode
|
||||
print 'headers =', headers
|
||||
print 'errmsg =', errmsg
|
||||
if headers:
|
||||
for header in headers.headers: print string.strip(header)
|
||||
print h.getfile().read()
|
||||
|
||||
if __name__ == '__main__':
|
||||
test()
|
||||
|
|
@ -0,0 +1,222 @@
|
|||
# Open an arbitrary URL
|
||||
#
|
||||
# See the following document for a tentative description of URLs:
|
||||
# Uniform Resource Locators Tim Berners-Lee
|
||||
# INTERNET DRAFT CERN
|
||||
# IETF URL Working Group 14 July 1993
|
||||
# draft-ietf-uri-url-01.txt
|
||||
#
|
||||
# The object returned by urlopen() will differ per protocol.
|
||||
# All you know is that is has methods read(), fileno(), close() and info().
|
||||
# The read(), fileno() and close() methods work like those of open files.
|
||||
# The info() method returns an rfc822.Message object which can be
|
||||
# used to query various info about the object, if available.
|
||||
# (rfc822.Message objects are queried with the getheader() method.)
|
||||
|
||||
import socket
|
||||
import regex
|
||||
import regsub
|
||||
import string
|
||||
import rfc822
|
||||
import ftplib
|
||||
|
||||
|
||||
# External interface -- use urlopen(file) as if it were open(file, 'r')
|
||||
def urlopen(url):
|
||||
url = string.strip(url)
|
||||
if url[:1] == '<' and url[-1:] == '>': url = string.strip(url[1:-1])
|
||||
if url[:4] == 'URL:': url = string.strip(url[4:])
|
||||
type, url = splittype(url)
|
||||
if not type: type = 'file'
|
||||
type = regsub.gsub('-', '_', type)
|
||||
try:
|
||||
func = eval('open_' + type)
|
||||
except NameError:
|
||||
raise IOError, ('url error', 'unknown url type', type)
|
||||
try:
|
||||
return func(url)
|
||||
except socket.error, msg:
|
||||
raise IOError, ('socket error', msg)
|
||||
|
||||
|
||||
# Each routine of the form open_<type> knows how to open that type of URL
|
||||
|
||||
# Use HTTP protocol
|
||||
def open_http(url):
|
||||
import httplib
|
||||
host, selector = splithost(url)
|
||||
h = httplib.HTTP(host)
|
||||
h.putrequest('GET', selector)
|
||||
errcode, errmsg, headers = h.getreply()
|
||||
if errcode == 200: return makefile(h.getfile(), headers)
|
||||
else: raise IOError, ('http error', errcode, errmsg, headers)
|
||||
|
||||
# Empty rfc822.Message object
|
||||
noheaders = rfc822.Message(open('/dev/null', 'r'))
|
||||
noheaders.fp.close() # Recycle file descriptor
|
||||
|
||||
# Use Gopher protocol
|
||||
def open_gopher(url):
|
||||
import gopherlib
|
||||
host, selector = splithost(url)
|
||||
type, selector = splitgophertype(selector)
|
||||
selector, query = splitquery(selector)
|
||||
if query: fp = gopherlib.send_query(selector, query, host)
|
||||
else: fp = gopherlib.send_selector(selector, host)
|
||||
return makefile(fp, noheaders)
|
||||
|
||||
# Use local file or FTP depending on form of URL
|
||||
localhost = socket.gethostbyname('localhost')
|
||||
thishost = socket.gethostbyname(socket.gethostname())
|
||||
def open_file(url):
|
||||
host, file = splithost(url)
|
||||
if not host: return makefile(open(file, 'r'), noheaders)
|
||||
host, port = splitport(host)
|
||||
if not port and socket.gethostbyname(host) in (localhost, thishost):
|
||||
try: fp = open(file, 'r')
|
||||
except IOError: fp = None
|
||||
if fp: return makefile(fp, noheaders)
|
||||
return open_ftp(url)
|
||||
|
||||
# Use FTP protocol
|
||||
ftpcache = {}
|
||||
ftperrors = (ftplib.error_reply,
|
||||
ftplib.error_temp,
|
||||
ftplib.error_perm,
|
||||
ftplib.error_proto)
|
||||
def open_ftp(url):
|
||||
host, file = splithost(url)
|
||||
host, port = splitport(host)
|
||||
host = socket.gethostbyname(host)
|
||||
if not port: port = ftplib.FTP_PORT
|
||||
key = (host, port)
|
||||
try:
|
||||
if not ftpcache.has_key(key):
|
||||
ftpcache[key] = ftpwrapper(host, port)
|
||||
return makefile(ftpcache[key].retrfile(file), noheaders)
|
||||
except ftperrors, msg:
|
||||
raise IOError, ('ftp error', msg)
|
||||
|
||||
|
||||
# Utility classes
|
||||
|
||||
# Class used to add an info() method to a file object
|
||||
class makefile:
|
||||
def __init__(self, fp, headers):
|
||||
self.fp = fp
|
||||
self.headers = headers
|
||||
self.read = self.fp.read
|
||||
self.fileno = self.fp.fileno
|
||||
self.close = self.fp.close
|
||||
def info(self):
|
||||
return self.headers
|
||||
|
||||
# Class used by open_ftp() for cache of open FTP connections
|
||||
class ftpwrapper:
|
||||
def __init__(self, host, port):
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.init()
|
||||
def init(self):
|
||||
self.ftp = ftplib.FTP()
|
||||
self.ftp.connect(self.host, self.port)
|
||||
self.ftp.login()
|
||||
def retrfile(self, file):
|
||||
try:
|
||||
self.ftp.voidcmd('TYPE I')
|
||||
except ftplib.all_errors:
|
||||
self.init()
|
||||
self.ftp.voidcmd('TYPE I')
|
||||
conn = None
|
||||
if file:
|
||||
try:
|
||||
cmd = 'RETR ' + file
|
||||
conn = self.ftp.transfercmd(cmd)
|
||||
except ftplib.error_perm, reason:
|
||||
if reason[:3] != '550':
|
||||
raise IOError, ('ftp error', reason)
|
||||
if not conn:
|
||||
# Try a directory listing
|
||||
if file: cmd = 'NLST ' + file
|
||||
else: cmd = 'NLST'
|
||||
conn = self.ftp.transfercmd(cmd)
|
||||
return fakefile(self.ftp, conn)
|
||||
|
||||
# Class used by ftpwrapper to handle response when transfer is complete
|
||||
class fakefile:
|
||||
def __init__(self, ftp, conn):
|
||||
self.ftp = ftp
|
||||
self.conn = conn
|
||||
self.fp = self.conn.makefile('r')
|
||||
self.read = self.fp.read
|
||||
self.fileno = self.fp.fileno
|
||||
def __del__(self):
|
||||
self.close()
|
||||
def close(self):
|
||||
self.conn = None
|
||||
self.fp = None
|
||||
self.read = None
|
||||
if self.ftp: self.ftp.voidresp()
|
||||
self.ftp = None
|
||||
|
||||
|
||||
# Utilities to split url parts into components:
|
||||
# splittype('type:opaquestring') --> 'type', 'opaquestring'
|
||||
# splithost('//host[:port]/path') --> 'host[:port]', '/path'
|
||||
# splitport('host:port') --> 'host', 'port'
|
||||
# splitquery('/path?query') --> '/path', 'query'
|
||||
# splittag('/path#tag') --> '/path', 'tag'
|
||||
# splitgophertype('/Xselector') --> 'X', 'selector'
|
||||
|
||||
typeprog = regex.compile('^\([^/:]+\):\(.*\)$')
|
||||
def splittype(url):
|
||||
if typeprog.match(url) >= 0: return typeprog.group(1, 2)
|
||||
return None, url
|
||||
|
||||
hostprog = regex.compile('^//\([^/]+\)\(.*\)$')
|
||||
def splithost(url):
|
||||
if hostprog.match(url) >= 0: return hostprog.group(1, 2)
|
||||
return None, url
|
||||
|
||||
portprog = regex.compile('^\(.*\):\([0-9]+\)$')
|
||||
def splitport(host):
|
||||
if portprog.match(host) >= 0: return portprog.group(1, 2)
|
||||
return host, None
|
||||
|
||||
queryprog = regex.compile('^\(.*\)\?\([^?]*\)$')
|
||||
def splitquery(url):
|
||||
if queryprog.match(url) >= 0: return queryprog.group(1, 2)
|
||||
return url, None
|
||||
|
||||
tagprog = regex.compile('^\(.*\)#\([^#]*\)$')
|
||||
def splittag(url):
|
||||
if tagprog.match(url) >= 0: return tagprog.group(1, 2)
|
||||
return url, None
|
||||
|
||||
def splitgophertype(selector):
|
||||
if selector[:1] == '/' and selector[1:2]:
|
||||
return selector[1], selector[2:]
|
||||
return None, selector
|
||||
|
||||
|
||||
# Test program
|
||||
def test():
|
||||
import sys
|
||||
args = sys.argv[1:]
|
||||
if not args:
|
||||
args = [
|
||||
'/etc/passwd',
|
||||
'file:/etc/passwd',
|
||||
'file://localhost/etc/passwd',
|
||||
'ftp://ftp.cwi.nl/etc/passwd',
|
||||
'gopher://gopher.cwi.nl/11/',
|
||||
'http://www.cwi.nl/index.html',
|
||||
]
|
||||
for arg in args:
|
||||
print '-'*10, arg, '-'*10
|
||||
print regsub.gsub('\r', '', urlopen(arg).read())
|
||||
print '-'*40
|
||||
|
||||
# Run test program when run as a script
|
||||
if __name__ == '__main__':
|
||||
test()
|
||||
Loading…
Reference in New Issue