mirror of https://github.com/python/cpython
363 lines
11 KiB
Python
Executable File
363 lines
11 KiB
Python
Executable File
#! /usr/bin/env python
|
|
#######################################################################
|
|
# Newslist $Revision$
|
|
#
|
|
# Syntax:
|
|
# newslist [ -a ]
|
|
#
|
|
# This is a program to create a directory full of HTML pages
|
|
# which between them contain links to all the newsgroups available
|
|
# on your server.
|
|
#
|
|
# The -a option causes a complete list of all groups to be read from
|
|
# the server rather than just the ones which have appeared since last
|
|
# execution. This recreates the local list from scratch. Use this on
|
|
# the first invocation of the program, and from time to time thereafter.
|
|
# When new groups are first created they may appear on your server as
|
|
# empty groups. By default, empty groups are ignored by the -a option.
|
|
# However, these new groups will not be created again, and so will not
|
|
# appear in the server's list of 'new groups' at a later date. Hence it
|
|
# won't appear until you do a '-a' after some articles have appeared.
|
|
#
|
|
# I should really keep a list of ignored empty groups and re-check them
|
|
# for articles on every run, but I haven't got around to it yet.
|
|
#
|
|
# This assumes an NNTP news feed.
|
|
#
|
|
# Feel free to copy, distribute and modify this code for
|
|
# non-commercial use. If you make any useful modifications, let me
|
|
# know!
|
|
#
|
|
# (c) Quentin Stafford-Fraser 1994
|
|
# fraser@europarc.xerox.com qs101@cl.cam.ac.uk
|
|
# #
|
|
#######################################################################
|
|
import sys, nntplib, marshal, time, os
|
|
|
|
#######################################################################
|
|
# Check these variables before running! #
|
|
|
|
# Top directory.
|
|
# Filenames which don't start with / are taken as being relative to this.
|
|
topdir = os.path.expanduser('~/newspage')
|
|
|
|
# The name of your NNTP host
|
|
# eg.
|
|
# newshost = 'nntp-serv.cl.cam.ac.uk'
|
|
# or use following to get the name from the NNTPSERVER environment
|
|
# variable:
|
|
# newshost = os.environ['NNTPSERVER']
|
|
newshost = 'news.example.com'
|
|
|
|
# The filename for a local cache of the newsgroup list
|
|
treefile = 'grouptree'
|
|
|
|
# The filename for descriptions of newsgroups
|
|
# I found a suitable one at ftp.uu.net in /uunet-info/newgroups.gz
|
|
# You can set this to '' if you don't wish to use one.
|
|
descfile = 'newsgroups'
|
|
|
|
# The directory in which HTML pages should be created
|
|
# eg.
|
|
# pagedir = '/usr/local/lib/html/newspage'
|
|
# pagedir = 'pages'
|
|
pagedir = topdir
|
|
|
|
# The html prefix which will refer to this directory
|
|
# eg.
|
|
# httppref = '/newspage/',
|
|
# or leave blank for relative links between pages: (Recommended)
|
|
# httppref = ''
|
|
httppref = ''
|
|
|
|
# The name of the 'root' news page in this directory.
|
|
# A .html suffix will be added.
|
|
rootpage = 'root'
|
|
|
|
# Set skipempty to 0 if you wish to see links to empty groups as well.
|
|
# Only affects the -a option.
|
|
skipempty = 1
|
|
|
|
# pagelinkicon can contain html to put an icon after links to
|
|
# further pages. This helps to make important links stand out.
|
|
# Set to '' if not wanted, or '...' is quite a good one.
|
|
pagelinkicon = '... <img src="http://pelican.cl.cam.ac.uk/icons/page.xbm"> '
|
|
|
|
# ---------------------------------------------------------------------
|
|
# Less important personal preferences:
|
|
|
|
# Sublistsize controls the maximum number of items the will appear as
|
|
# an indented sub-list before the whole thing is moved onto a different
|
|
# page. The smaller this is, the more pages you will have, but the
|
|
# shorter each will be.
|
|
sublistsize = 4
|
|
|
|
# That should be all. #
|
|
#######################################################################
|
|
|
|
for dir in os.curdir, os.environ['HOME']:
|
|
rcfile = os.path.join(dir, '.newslistrc.py')
|
|
if os.path.exists(rcfile):
|
|
print rcfile
|
|
execfile(rcfile)
|
|
break
|
|
|
|
from nntplib import NNTP
|
|
from stat import *
|
|
|
|
rcsrev = '$Revision$'
|
|
rcsrev = ' '.join(filter(lambda s: '$' not in s, rcsrev.split()))
|
|
desc = {}
|
|
|
|
# Make (possibly) relative filenames into absolute ones
|
|
treefile = os.path.join(topdir,treefile)
|
|
descfile = os.path.join(topdir,descfile)
|
|
page = os.path.join(topdir,pagedir)
|
|
|
|
# First the bits for creating trees ---------------------------
|
|
|
|
# Addtotree creates/augments a tree from a list of group names
|
|
def addtotree(tree, groups):
|
|
print 'Updating tree...'
|
|
for i in groups:
|
|
parts = i.split('.')
|
|
makeleaf(tree, parts)
|
|
|
|
# Makeleaf makes a leaf and the branch leading to it if necessary
|
|
def makeleaf(tree,path):
|
|
j = path[0]
|
|
l = len(path)
|
|
|
|
if not tree.has_key(j):
|
|
tree[j] = {}
|
|
if l == 1:
|
|
tree[j]['.'] = '.'
|
|
if l > 1:
|
|
makeleaf(tree[j],path[1:])
|
|
|
|
# Then the bits for outputting trees as pages ----------------
|
|
|
|
# Createpage creates an HTML file named <root>.html containing links
|
|
# to those groups beginning with <root>.
|
|
|
|
def createpage(root, tree, p):
|
|
filename = os.path.join(pagedir, root+'.html')
|
|
if root == rootpage:
|
|
detail = ''
|
|
else:
|
|
detail = ' under ' + root
|
|
with open(filename, 'w') as f:
|
|
# f.write('Content-Type: text/html\n')
|
|
f.write('<html>\n<head>\n')
|
|
f.write('<title>Newsgroups available%s</title>\n' % detail)
|
|
f.write('</head>\n<body>\n')
|
|
f.write('<h1>Newsgroups available%s</h1>\n' % detail)
|
|
f.write('<a href="%s%s.html">Back to top level</a><p>\n' %
|
|
(httppref, rootpage))
|
|
printtree(f, tree, 0, p)
|
|
f.write('\n<p>')
|
|
f.write("<i>This page automatically created by 'newslist' v. %s." %
|
|
rcsrev)
|
|
f.write(time.ctime(time.time()) + '</i>\n')
|
|
f.write('</body>\n</html>\n')
|
|
|
|
# Printtree prints the groups as a bulleted list. Groups with
|
|
# more than <sublistsize> subgroups will be put on a separate page.
|
|
# Other sets of subgroups are just indented.
|
|
|
|
def printtree(f, tree, indent, p):
|
|
l = len(tree)
|
|
|
|
if l > sublistsize and indent > 0:
|
|
# Create a new page and a link to it
|
|
f.write('<li><b><a href="%s%s.html">' % (httppref, p[1:]))
|
|
f.write(p[1:] + '.*')
|
|
f.write('</a></b>%s\n' % pagelinkicon)
|
|
createpage(p[1:], tree, p)
|
|
return
|
|
|
|
kl = tree.keys()
|
|
|
|
if l > 1:
|
|
kl.sort()
|
|
if indent > 0:
|
|
# Create a sub-list
|
|
f.write('<li>%s\n<ul>' % p[1:])
|
|
else:
|
|
# Create a main list
|
|
f.write('<ul>')
|
|
indent = indent + 1
|
|
|
|
for i in kl:
|
|
if i == '.':
|
|
# Output a newsgroup
|
|
f.write('<li><a href="news:%s">%s</a> ' % (p[1:], p[1:]))
|
|
if p[1:] in desc:
|
|
f.write(' <i>%s</i>\n' % desc[p[1:]])
|
|
else:
|
|
f.write('\n')
|
|
else:
|
|
# Output a hierarchy
|
|
printtree(f, tree[i], indent, p+'.'+i)
|
|
|
|
if l > 1:
|
|
f.write('\n</ul>')
|
|
|
|
# Reading descriptions file ---------------------------------------
|
|
|
|
# This returns a dict mapping group name to its description
|
|
|
|
def readdesc(descfile):
|
|
global desc
|
|
desc = {}
|
|
|
|
if descfile == '':
|
|
return
|
|
|
|
try:
|
|
with open(descfile, 'r') as d:
|
|
print 'Reading descriptions...'
|
|
for l in d:
|
|
bits = l.split()
|
|
try:
|
|
grp = bits[0]
|
|
dsc = ' '.join(bits[1:])
|
|
if len(dsc) > 1:
|
|
desc[grp] = dsc
|
|
except IndexError:
|
|
pass
|
|
except IOError:
|
|
print 'Failed to open description file ' + descfile
|
|
return
|
|
|
|
# Check that ouput directory exists, ------------------------------
|
|
# and offer to create it if not
|
|
|
|
def checkopdir(pagedir):
|
|
if not os.path.isdir(pagedir):
|
|
print 'Directory %s does not exist.' % pagedir
|
|
print 'Shall I create it for you? (y/n)'
|
|
if sys.stdin.readline()[0] == 'y':
|
|
try:
|
|
os.mkdir(pagedir, 0777)
|
|
except:
|
|
print 'Sorry - failed!'
|
|
sys.exit(1)
|
|
else:
|
|
print 'OK. Exiting.'
|
|
sys.exit(1)
|
|
|
|
# Read and write current local tree ----------------------------------
|
|
|
|
def readlocallist(treefile):
|
|
print 'Reading current local group list...'
|
|
tree = {}
|
|
try:
|
|
treetime = time.localtime(os.stat(treefile)[ST_MTIME])
|
|
except:
|
|
print '\n*** Failed to open local group cache '+treefile
|
|
print 'If this is the first time you have run newslist, then'
|
|
print 'use the -a option to create it.'
|
|
sys.exit(1)
|
|
treedate = '%02d%02d%02d' % (treetime[0] % 100, treetime[1], treetime[2])
|
|
try:
|
|
with open(treefile, 'rb') as dump:
|
|
tree = marshal.load(dump)
|
|
except IOError:
|
|
print 'Cannot open local group list ' + treefile
|
|
return (tree, treedate)
|
|
|
|
def writelocallist(treefile, tree):
|
|
try:
|
|
with open(treefile, 'wb') as dump:
|
|
groups = marshal.dump(tree, dump)
|
|
print 'Saved list to %s\n' % treefile
|
|
except:
|
|
print 'Sorry - failed to write to local group cache', treefile
|
|
print 'Does it (or its directory) have the correct permissions?'
|
|
sys.exit(1)
|
|
|
|
# Return list of all groups on server -----------------------------
|
|
|
|
def getallgroups(server):
|
|
print 'Getting list of all groups...'
|
|
treedate = '010101'
|
|
info = server.list()[1]
|
|
groups = []
|
|
print 'Processing...'
|
|
if skipempty:
|
|
print '\nIgnoring following empty groups:'
|
|
for i in info:
|
|
grpname = i[0].split()[0]
|
|
if skipempty and int(i[1]) < int(i[2]):
|
|
print grpname + ' ',
|
|
else:
|
|
groups.append(grpname)
|
|
print '\n'
|
|
if skipempty:
|
|
print '(End of empty groups)'
|
|
return groups
|
|
|
|
# Return list of new groups on server -----------------------------
|
|
|
|
def getnewgroups(server, treedate):
|
|
print 'Getting list of new groups since start of %s...' % treedate,
|
|
info = server.newgroups(treedate, '000001')[1]
|
|
print 'got %d.' % len(info)
|
|
print 'Processing...',
|
|
groups = []
|
|
for i in info:
|
|
grpname = i.split()[0]
|
|
groups.append(grpname)
|
|
print 'Done'
|
|
return groups
|
|
|
|
# Now the main program --------------------------------------------
|
|
|
|
def main():
|
|
tree = {}
|
|
|
|
# Check that the output directory exists
|
|
checkopdir(pagedir)
|
|
|
|
try:
|
|
print 'Connecting to %s...' % newshost
|
|
if sys.version[0] == '0':
|
|
s = NNTP.init(newshost)
|
|
else:
|
|
s = NNTP(newshost)
|
|
connected = True
|
|
except (nntplib.error_temp, nntplib.error_perm), x:
|
|
print 'Error connecting to host:', x
|
|
print 'I\'ll try to use just the local list.'
|
|
connected = False
|
|
|
|
# If -a is specified, read the full list of groups from server
|
|
if connected and len(sys.argv) > 1 and sys.argv[1] == '-a':
|
|
groups = getallgroups(s)
|
|
|
|
# Otherwise just read the local file and then add
|
|
# groups created since local file last modified.
|
|
else:
|
|
|
|
(tree, treedate) = readlocallist(treefile)
|
|
if connected:
|
|
groups = getnewgroups(s, treedate)
|
|
|
|
if connected:
|
|
addtotree(tree, groups)
|
|
writelocallist(treefile,tree)
|
|
|
|
# Read group descriptions
|
|
readdesc(descfile)
|
|
|
|
print 'Creating pages...'
|
|
createpage(rootpage, tree, '')
|
|
print 'Done'
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|
|
# That's all folks
|
|
######################################################################
|