Added __del__(), close(), cleanup(), retrieve() and open_local_file()

Lib/urlopen.py

@@ -32,6 +32,11 @@ def urlopen(url):
 	if not _urlopener:
 		_urlopener = URLopener()
 	return _urlopener.open(url)
+def urlretrieve(url):
+	global _urlopener
+	if not _urlopener:
+		_urlopener = URLopener()
+	return _urlopener.retrieve(url)
 
 
 # Class to open URLs.
@@ -44,10 +49,25 @@ class URLopener:
 	def __init__(self):
 		self.addheaders = []
 		self.ftpcache = ftpcache
+		self.tempfiles = []
 		# Undocumented feature: you can use a different
 		# ftp cache by assigning to the .ftpcache member;
 		# in case you want logically independent URL openers
 
+	def __del__(self):
+		self.close()
+
+	def close(self):
+		self.cleanup()
+
+	def cleanup(self):
+		import os
+		for tfn in self.tempfiles:
+			try:
+				os.unlink(tfn)
+			except os.error:
+				pass
+
 	# Add a header to be used by the HTTP interface only
 	# e.g. u.addheader('Accept', 'sound/basic')
 	def addheader(self, *args):
@@ -56,13 +76,8 @@ class URLopener:
 	# External interface
 	# Use URLopener().open(file) instead of open(file, 'r')
 	def open(self, url):
-		import string
+		type, url = splittype(unwrap(url))
-		url = string.strip(url)
+ 		if not type: type = 'file'
-		if url[:1] == '<' and url[-1:] == '>':
-			url = string.strip(url[1:-1])
-		if url[:4] == 'URL:': url = string.strip(url[4:])
-		type, url = splittype(url)
-		if not type: type = 'file'
 		name = 'open_' + type
 		if '-' in name:
 			import regsub
@@ -75,6 +90,32 @@ class URLopener:
 		except socket.error, msg:
 			raise IOError, ('socket error', msg)
 
+	# External interface
+	# retrieve(url) returns (filename, None) for a local object
+	# or (tempfilename, headers) for a remote object
+	def retrieve(self, url):
+		type, url1 = splittype(unwrap(url))
+		if not type or type == 'file':
+			try:
+				fp = self.open_local_file(url1)
+				return splithost(url1)[1], None
+			except IOError, msg:
+				pass
+		fp = self.open(url)
+		import tempfile
+		tfn = tempfile.mktemp()
+		self.tempfiles.append(tfn)
+		tfp = open(tfn, 'w')
+		bs = 1024*8
+		block = fp.read(bs)
+		while block:
+			tfp.write(block)
+			block = fp.read(bs)
+		headers = fp.info()
+		fp.close()
+		tfp.close()
+		return tfn, headers
+
 	# Each method named open_<type> knows how to open that type of URL
 
 	# Use HTTP protocol
@@ -100,15 +141,20 @@ class URLopener:
 
 	# Use local file or FTP depending on form of URL
 	def open_file(self, url):
+		try:
+			return self.open_local_file(url)
+		except IOError:
+			return self.open_ftp(url)
+
+	# Use local file
+	def open_local_file(self, url):
 		host, file = splithost(url)
 		if not host: return addinfo(open(file, 'r'), noheaders())
 		host, port = splitport(host)
 		if not port and socket.gethostbyname(host) in (
 			  localhost(), thishost()):
-			try: fp = open(file, 'r')
+			return addinfo(open(file, 'r'), noheaders())
-			except IOError: fp = None
+		raise IOError, ('local file error', 'not on local host')
-			if fp: return addinfo(fp, noheaders())
-		return self.open_ftp(url)
 
 	# Use FTP protocol
 	def open_ftp(self, url):
@@ -199,8 +245,8 @@ class ftpwrapper:
 					raise IOError, ('ftp error', reason)
 		if not conn:
 			# Try a directory listing
-			if file: cmd = 'NLST ' + file
+			if file: cmd = 'LIST ' + file
-			else: cmd = 'NLST'
+			else: cmd = 'LIST'
 			conn = self.ftp.transfercmd(cmd)
 		return addclosehook(conn.makefile('r'), self.ftp.voidresp)
 
@@ -215,6 +261,11 @@ class addbase:
 	def __del__(self):
 		self.close()
 	def close(self):
+		self.read = None
+		self.readline = None
+		self.readlines = None
+		self.fileno = None
+		self.fp.close()
 		self.fp = None
 
 # Class to add a close hook to an open file
@@ -227,7 +278,8 @@ class addclosehook(addbase):
 		if self.closehook:
 			apply(self.closehook, self.hookargs)
 			self.closehook = None
-		self.fp = None
+			self.hookargs = None
+		addbase.close(self)
 
 # class to add an info() method to an open file
 class addinfo(addbase):
@@ -239,6 +291,7 @@ class addinfo(addbase):
 
 
 # Utilities to parse URLs:
+# unwrap('<URL:type//host/path>') --> 'type//host/path'
 # splittype('type:opaquestring') --> 'type', 'opaquestring'
 # splithost('//host[:port]/path') --> 'host[:port]', '/path'
 # splitport('host:port') --> 'host', 'port'
@@ -246,6 +299,14 @@ class addinfo(addbase):
 # splittag('/path#tag') --> '/path', 'tag'
 # splitgophertype('/Xselector') --> 'X', 'selector'
 
+def unwrap(url):
+	import string
+	url = string.strip(url)
+	if url[:1] == '<' and url[-1:] == '>':
+		url = string.strip(url[1:-1])
+	if url[:4] == 'URL:': url = string.strip(url[4:])
+	return url
+
 _typeprog = regex.compile('^\([^/:]+\):\(.*\)$')
 def splittype(url):
 	if _typeprog.match(url) >= 0: return _typeprog.group(1, 2)
@@ -291,10 +352,21 @@ def test():
 			'gopher://gopher.cwi.nl/11/',
 			'http://www.cwi.nl/index.html',
 			]
-	for arg in args:
+	try:
-		print '-'*10, arg, '-'*10
+		for url in args:
-		print regsub.gsub('\r', '', urlopen(arg).read())
+			print '-'*10, url, '-'*10
-	print '-'*40
+			fn, h = urlretrieve(url)
+			print fn, h
+			if h:
+				print '======'
+				for k in h.keys(): print k + ':', h[k]
+				print '======'
+			fp = open(fn, 'r')
+			data = fp.read()
+			print regsub.gsub('\r', '', data)
+		print '-'*40
+	finally:
+		_urlopener.cleanup()
 
 # Run test program when run as a script
 if __name__ == '__main__':