commit -- why not

This commit is contained in:
Guido van Rossum 1995-04-10 11:40:52 +00:00
parent 21bc15b73d
commit 07a272d9de
5 changed files with 775 additions and 0 deletions

322
Demo/pdist/FSProxy.py Executable file
View File

@ -0,0 +1,322 @@
"""File System Proxy.
Provide an OS-neutral view on a file system, locally or remotely.
The functionality is geared towards implementing some sort of
rdist-like utility between a Mac and a UNIX system.
The module defines three classes:
FSProxyLocal -- used for local access
FSProxyServer -- used on the server side of remote access
FSProxyClient -- used on the client side of remote access
The remote classes are instantiated with an IP address and an optional
verbosity flag.
"""
import server
import client
import md5
import os
import fnmatch
from stat import *
import time
import fnmatch
if os.name == 'mac':
import macfs
maxnamelen = 31
else:
macfs = None
maxnamelen = 255
skipnames = (os.curdir, os.pardir)
class FSProxyLocal:
def __init__(self):
self._dirstack = []
self._ignore = ['*.pyc'] + self._readignore()
def _close(self):
while self._dirstack:
self.back()
def _readignore(self):
file = self._hide('ignore')
try:
f = open(file)
except IOError:
file = self._hide('synctree.ignorefiles')
try:
f = open(file)
except IOError:
return []
ignore = []
while 1:
line = f.readline()
if not line: break
if line[-1] == '\n': line = line[:-1]
ignore.append(line)
f.close()
return ignore
def _hidden(self, name):
if os.name == 'mac':
return name[0] == '(' and name[-1] == ')'
else:
return name[0] == '.'
def _hide(self, name):
if os.name == 'mac':
return '(%s)' % name
else:
return '.%s' % name
def visible(self, name):
if len(name) > maxnamelen: return 0
if name[-1] == '~': return 0
if name in skipnames: return 0
if self._hidden(name): return 0
head, tail = os.path.split(name)
if head or not tail: return 0
if macfs:
if os.path.exists(name) and not os.path.isdir(name):
try:
fs = macfs.FSSpec(name)
c, t = fs.GetCreatorType()
if t != 'TEXT': return 0
except macfs.error, msg:
print "***", name, msg
return 0
else:
if os.path.islink(name): return 0
if '\0' in open(name, 'rb').read(512): return 0
for ign in self._ignore:
if fnmatch.fnmatch(name, ign): return 0
return 1
def check(self, name):
if not self.visible(name):
raise os.error, "protected name %s" % repr(name)
def checkfile(self, name):
self.check(name)
if not os.path.isfile(name):
raise os.error, "not a plain file %s" % repr(name)
def pwd(self):
return os.getcwd()
def cd(self, name):
self.check(name)
save = os.getcwd(), self._ignore
os.chdir(name)
self._dirstack.append(save)
self._ignore = self._ignore + self._readignore()
def back(self):
if not self._dirstack:
raise os.error, "empty directory stack"
dir, ignore = self._dirstack[-1]
os.chdir(dir)
del self._dirstack[-1]
self._ignore = ignore
def _filter(self, files, pat = None):
if pat:
def keep(name, pat = pat):
return fnmatch.fnmatch(name, pat)
files = filter(keep, files)
files = filter(self.visible, files)
files.sort()
return files
def list(self, pat = None):
files = os.listdir(os.curdir)
return self._filter(files, pat)
def listfiles(self, pat = None):
files = os.listdir(os.curdir)
files = filter(os.path.isfile, files)
return self._filter(files, pat)
def listsubdirs(self, pat = None):
files = os.listdir(os.curdir)
files = filter(os.path.isdir, files)
return self._filter(files, pat)
def exists(self, name):
return self.visible(name) and os.path.exists(name)
def isdir(self, name):
return self.visible(name) and os.path.isdir(name)
def islink(self, name):
return self.visible(name) and os.path.islink(name)
def isfile(self, name):
return self.visible(name) and os.path.isfile(name)
def sum(self, name):
self.checkfile(name)
BUFFERSIZE = 1024*8
f = open(name)
sum = md5.new()
while 1:
buffer = f.read(BUFFERSIZE)
if not buffer:
break
sum.update(buffer)
return sum.digest()
def size(self, name):
self.checkfile(name)
return os.stat(name)[ST_SIZE]
def mtime(self, name):
self.checkfile(name)
return time.localtime(os.stat(name)[ST_MTIME])
def stat(self, name):
self.checkfile(name)
size = os.stat(name)[ST_SIZE]
mtime = time.localtime(os.stat(name)[ST_MTIME])
return size, mtime
def info(self, name):
sum = self.sum(name)
size = os.stat(name)[ST_SIZE]
mtime = time.localtime(os.stat(name)[ST_MTIME])
return sum, size, mtime
def _list(self, function, list):
if list is None:
list = self.listfiles()
res = []
for name in list:
try:
res.append((name, function(name)))
except (os.error, IOError):
res.append((name, None))
return res
def sumlist(self, list = None):
return self._list(self.sum, list)
def statlist(self, list = None):
return self._list(self.stat, list)
def mtimelist(self, list = None):
return self._list(self.mtime, list)
def sizelist(self, list = None):
return self._list(self.size, list)
def infolist(self, list = None):
return self._list(self.info, list)
def _dict(self, function, list):
if list is None:
list = self.listfiles()
dict = {}
for name in list:
try:
dict[name] = function(name)
except (os.error, IOError):
pass
return dict
def sumdict(self, list = None):
return self.dict(self.sum, list)
def sizedict(self, list = None):
return self.dict(self.size, list)
def mtimedict(self, list = None):
return self.dict(self.mtime, list)
def statdict(self, list = None):
return self.dict(self.stat, list)
def infodict(self, list = None):
return self._dict(self.info, list)
def read(self, name, offset = 0, length = -1):
self.checkfile(name)
f = open(name)
f.seek(offset)
if length == 0:
data = ''
elif length < 0:
data = f.read()
else:
data = f.read(length)
f.close()
return data
def create(self, name):
self.check(name)
if os.path.exists(name):
self.checkfile(name)
bname = name + '~'
try:
os.unlink(bname)
except os.error:
pass
os.rename(name, bname)
f = open(name, 'w')
f.close()
def write(self, name, data, offset = 0):
self.checkfile(name)
f = open(name, 'r+')
f.seek(offset)
f.write(data)
f.close()
def mkdir(self, name):
self.check(name)
os.mkdir(name, 0777)
def rmdir(self, name):
self.check(name)
os.rmdir(name)
class FSProxyServer(FSProxyLocal, server.Server):
def __init__(self, address, verbose = server.VERBOSE):
FSProxyLocal.__init__(self)
server.Server.__init__(self, address, verbose)
def _close(self):
server.Server._close(self)
FSProxyLocal._close(self)
def _serve(self):
server.Server._serve(self)
# Retreat into start directory
while self._dirstack: self.back()
class FSProxyClient(client.Client):
def __init__(self, address, verbose = client.VERBOSE):
client.Client.__init__(self, address, verbose)
def test():
import string
import sys
if sys.argv[1:]:
port = string.atoi(sys.argv[1])
else:
port = 4127
proxy = FSProxyServer(('', port))
proxy._serverloop()
if __name__ == '__main__':
test()

133
Demo/pdist/client.py Executable file
View File

@ -0,0 +1,133 @@
"""RPC Client module."""
import sys
import socket
import pickle
import __builtin__
import os
# Default verbosity (0 = silent, 1 = print connections, 2 = print requests too)
VERBOSE = 1
class Client:
"""RPC Client class. No need to derive a class -- it's fully generic."""
def __init__(self, address, verbose = VERBOSE):
if type(address) == type(0):
address = ('', address)
self._address = address
self._verbose = verbose
if self._verbose: print "Connecting to %s ..." % repr(address)
self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._socket.connect(address)
if self._verbose: print "Connected."
self._lastid = 0 # Last id for which a reply has been received
self._nextid = 1 # Id of next request
self._replies = {} # Unprocessed replies
self._rf = self._socket.makefile('r')
self._wf = self._socket.makefile('w')
self._methods = self._call('.methods')
def __del__(self):
self._close()
def _close(self):
if self._rf: self._rf.close()
self._rf = None
if self._wf: self._wf.close()
self._wf = None
if self._socket: self._socket.close()
self._socket = None
def __getattr__(self, name):
if name in self._methods:
method = _stub(self, name)
setattr(self, name, method) # XXX circular reference
return method
raise AttributeError, name
def _setverbose(self, verbose):
self._verbose = verbose
def _call(self, name, *args):
return self._vcall(name, args)
def _vcall(self, name, args):
return self._recv(self._vsend(name, args))
def _send(self, name, *args):
return self._vsend(name, args)
def _send_noreply(self, name, *args):
return self._vsend(name, args, 0)
def _vsend_noreply(self, name, args):
return self._vsend(name, args, 0)
def _vsend(self, name, args, wantreply = 1):
id = self._nextid
self._nextid = id+1
if not wantreply: id = -id
request = (name, args, id)
if self._verbose > 1: print "sending request: %s" % repr(request)
wp = pickle.Pickler(self._wf)
wp.dump(request)
return id
def _recv(self, id):
exception, value, rid = self._vrecv(id)
if rid != id:
raise RuntimeError, "request/reply id mismatch: %d/%d" % (id, rid)
if exception is None:
return value
x = exception
if hasattr(__builtin__, exception):
x = getattr(__builtin__, exception)
elif exception in ('posix.error', 'mac.error'):
x = os.error
if x == exception:
exception = x
raise exception, value
def _vrecv(self, id):
self._flush()
if self._replies.has_key(id):
if self._verbose > 1: print "retrieving previous reply, id = %d" % id
reply = self._replies[id]
del self._replies[id]
return reply
aid = abs(id)
while 1:
if self._verbose > 1: print "waiting for reply, id = %d" % id
rp = pickle.Unpickler(self._rf)
reply = rp.load()
del rp
if self._verbose > 1: print "got reply: %s" % repr(reply)
rid = reply[2]
arid = abs(rid)
if arid == aid:
if self._verbose > 1: print "got it"
return reply
self._replies[rid] = reply
if arid > aid:
if self._verbose > 1: print "got higher id, assume all ok"
return (None, None, id)
def _flush(self):
self._wf.flush()
class _stub:
"""Helper class for Client -- each instance serves as a method of the client."""
def __init__(self, client, name):
self._client = client
self._name = name
def __call__(self, *args):
return self._client._vcall(self._name, args)

185
Demo/pdist/cmptree.py Executable file
View File

@ -0,0 +1,185 @@
"""Compare local and remote dictionaries and transfer differing files -- like rdist."""
import sys
from repr import repr
import FSProxy
import time
import os
def main():
pwd = os.getcwd()
s = raw_input("chdir [%s] " % pwd)
if s:
os.chdir(s)
pwd = os.getcwd()
host = ask("host", 'voorn.cwi.nl')
port = 4127
verbose = 1
mode = ''
print """\
Mode should be a string of characters, indicating what to do with differences.
r - read different files to local file system
w - write different files to remote file system
c - create new files, either remote or local
d - delete disappearing files, either remote or local
"""
s = raw_input("mode [%s] " % mode)
if s: mode = s
address = (host, port)
t1 = time.time()
local = FSProxy.FSProxyLocal()
remote = FSProxy.FSProxyClient(address, verbose)
compare(local, remote, mode)
remote._close()
local._close()
t2 = time.time()
dt = t2-t1
mins, secs = divmod(dt, 60)
print mins, "minutes and", secs, "seconds"
raw_input("[Return to exit] ")
def ask(prompt, default):
s = raw_input("%s [%s] " % (prompt, default))
return s or default
def askint(prompt, default):
s = raw_input("%s [%s] " % (prompt, str(default)))
if s: return string.atoi(s)
return default
def compare(local, remote, mode):
print
print "PWD =", `os.getcwd()`
sums_id = remote._send('sumlist')
subdirs_id = remote._send('listsubdirs')
remote._flush()
print "calculating local sums ..."
lsumdict = {}
for name, info in local.sumlist():
lsumdict[name] = info
print "getting remote sums ..."
sums = remote._recv(sums_id)
print "got", len(sums)
rsumdict = {}
for name, rsum in sums:
rsumdict[name] = rsum
if not lsumdict.has_key(name):
print `name`, "only remote"
if 'r' in mode and 'c' in mode:
recvfile(local, remote, name)
else:
lsum = lsumdict[name]
if lsum != rsum:
print `name`,
rmtime = remote.mtime(name)
lmtime = local.mtime(name)
if rmtime > lmtime:
print "remote newer",
if 'r' in mode:
recvfile(local, remote, name)
elif lmtime > rmtime:
print "local newer",
if 'w' in mode:
sendfile(local, remote, name)
else:
print "same mtime but different sum?!?!",
print
for name in lsumdict.keys():
if not rsumdict.keys():
print `name`, "only locally",
fl()
if 'w' in mode and 'c' in mode:
sendfile(local, remote, name)
elif 'r' in mode and 'd' in mode:
os.unlink(name)
print "removed."
print
print "gettin subdirs ..."
subdirs = remote._recv(subdirs_id)
common = []
for name in subdirs:
if local.isdir(name):
print "Common subdirectory", repr(name)
common.append(name)
else:
print "Remote subdirectory", repr(name), "not found locally"
lsubdirs = local.listsubdirs()
for name in lsubdirs:
if name not in subdirs:
print "Local subdirectory", repr(name), "not found remotely"
for name in common:
print "Entering subdirectory", repr(name)
local.cd(name)
remote.cd(name)
compare(local, remote, mode)
remote.back()
local.back()
def sendfile(local, remote, name):
try:
remote.create(name)
except (IOError, os.error), msg:
print "cannot create:", msg
return
print "sending ...",
fl()
data = open(name).read()
t1 = time.time()
remote._send_noreply('write', name, data)
remote._flush()
t2 = time.time()
dt = t2-t1
print len(data), "bytes in", t2-t1, "seconds",
if dt:
print "i.e.", len(data)/dt, "bytes/sec",
print
def recvfile(local, remote, name):
try:
local.create(name)
except (IOError, os.error), msg:
print "cannot create:", msg
return
print "receiving ...",
fl()
f = open(name, 'w')
t1 = time.time()
length = 4*1024
offset = 0
id = remote._send('read', name, offset, length)
remote._flush()
while 1:
newoffset = offset + length
newid = remote._send('read', name, newoffset, length)
data = remote._recv(id)
id = newid
if not data: break
f.seek(offset)
f.write(data)
offset = newoffset
size = f.tell()
t2 = time.time()
f.close()
dt = t2-t1
print size, "bytes in", dt, "seconds",
if dt:
print "i.e.", int(size/dt), "bytes/sec",
print
remote._recv(id) # ignored
def fl():
sys.stdout.flush()
if __name__ == '__main__':
main()

111
Demo/pdist/server.py Executable file
View File

@ -0,0 +1,111 @@
"""RPC Server module."""
import sys
import socket
import pickle
from fnmatch import fnmatch
from repr import repr
# Default verbosity (0 = silent, 1 = print connections, 2 = print requests too)
VERBOSE = 1
class Server:
"""RPC Server class. Derive a class to implement a particular service."""
def __init__(self, address, verbose = VERBOSE):
if type(address) == type(0):
address = ('', address)
self._address = address
self._verbose = verbose
self._socket = None
self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._socket.bind(address)
self._socket.listen(1)
self._listening = 1
def _setverbose(self, verbose):
self._verbose = verbose
def __del__(self):
self._close()
def _close(self):
self._listening = 0
if self._socket:
self._socket.close()
self._socket = None
def _serverloop(self):
while self._listening:
self._serve()
def _serve(self):
if self._verbose: print "Wait for connection ..."
conn, address = self._socket.accept()
if self._verbose: print "Accepted connection from %s" % repr(address)
if not self._verify(conn, address):
print "*** Connection from %s refused" % repr(address)
conn.close()
return
rf = conn.makefile('r')
wf = conn.makefile('w')
ok = 1
while ok:
wf.flush()
if self._verbose > 1: print "Wait for next request ..."
ok = self._dorequest(rf, wf)
_valid = ['192.16.201.*', '192.16.197.*']
def _verify(self, conn, address):
host, port = address
for pat in self._valid:
if fnmatch(host, pat): return 1
return 0
def _dorequest(self, rf, wf):
rp = pickle.Unpickler(rf)
try:
request = rp.load()
except EOFError:
return 0
if self._verbose > 1: print "Got request: %s" % repr(request)
try:
methodname, args, id = request
if '.' in methodname:
reply = (None, self._special(methodname, args), id)
elif methodname[0] == '_':
raise NameError, "illegal method name %s" % repr(methodname)
else:
method = getattr(self, methodname)
reply = (None, apply(method, args), id)
except:
reply = (sys.exc_type, sys.exc_value, id)
if id < 0 and reply[:2] == (None, None):
if self._verbose > 1: print "Suppress reply"
return 1
if self._verbose > 1: print "Send reply: %s" % repr(reply)
wp = pickle.Pickler(wf)
wp.dump(reply)
return 1
def _special(self, methodname, args):
if methodname == '.methods':
if not hasattr(self, '_methods'):
self._methods = tuple(self._listmethods())
return self._methods
raise NameError, "unrecognized special method name %s" % repr(methodname)
def _listmethods(self, cl=None):
if not cl: cl = self.__class__
names = cl.__dict__.keys()
names = filter(lambda x: x[0] != '_', names)
names.sort()
for base in cl.__bases__:
basenames = self._listmethods(base)
basenames = filter(lambda x, names=names: x not in names, basenames)
names[len(names):] = basenames
return names

24
Demo/pdist/sumtree.py Executable file
View File

@ -0,0 +1,24 @@
import time
import FSProxy
def main():
t1 = time.time()
#proxy = FSProxy.FSProxyClient(('voorn.cwi.nl', 4127))
proxy = FSProxy.FSProxyLocal()
sumtree(proxy)
proxy._close()
t2 = time.time()
print t2-t1, "seconds"
raw_input("[Return to exit] ")
def sumtree(proxy):
print "PWD =", proxy.pwd()
files = proxy.listfiles()
proxy.infolist(files)
subdirs = proxy.listsubdirs()
for name in subdirs:
proxy.cd(name)
sumtree(proxy)
proxy.back()
main()