#! /usr/bin/env python######################################################################## Newslist  $Revision: 37320 $## Syntax:#    newslist [ -a ]## This is a program to create a directory full of HTML pages# which between them contain links to all the newsgroups available# on your server.## The -a option causes a complete list of all groups to be read from# the server rather than just the ones which have appeared since last# execution. This recreates the local list from scratch. Use this on# the first invocation of the program, and from time to time thereafter.#   When new groups are first created they may appear on your server as# empty groups. By default, empty groups are ignored by the -a option.# However, these new groups will not be created again, and so will not# appear in the server's list of 'new groups' at a later date. Hence it# won't appear until you do a '-a' after some articles have appeared.## I should really keep a list of ignored empty groups and re-check them# for articles on every run, but I haven't got around to it yet.## This assumes an NNTP news feed.## Feel free to copy, distribute and modify this code for# non-commercial use. If you make any useful modifications, let me# know!## (c) Quentin Stafford-Fraser 1994# fraser@europarc.xerox.com                     qs101@cl.cam.ac.uk#                                                                     ########################################################################import sys,nntplib, string, marshal, time, os, posix, string ######################################################################## Check these variables before running!                               # # Top directory.# Filenames which don't start with / are taken as being relative to this.topdir='/anfs/qsbigdisc/web/html/newspage' # The name of your NNTP host# eg.#    newshost = 'nntp-serv.cl.cam.ac.uk'# or use following to get the name from the NNTPSERVER environment# variable:#    newshost = posix.environ['NNTPSERVER']newshost = 'nntp-serv.cl.cam.ac.uk' # The filename for a local cache of the newsgroup listtreefile = 'grouptree' # The filename for descriptions of newsgroups# I found a suitable one at ftp.uu.net in /uunet-info/newgroups.gz# You can set this to '' if you don't wish to use one.descfile = 'newsgroups' # The directory in which HTML pages should be created# eg.#   pagedir  = '/usr/local/lib/html/newspage'#   pagedir  = 'pages'pagedir  = topdir # The html prefix which will refer to this directory# eg.#   httppref = '/newspage/',# or leave blank for relative links between pages: (Recommended)#   httppref = ''httppref = '' # The name of the 'root' news page in this directory.# A .html suffix will be added.rootpage = 'root' # Set skipempty to 0 if you wish to see links to empty groups as well.# Only affects the -a option.skipempty = 1 # pagelinkicon can contain html to put an icon after links to# further pages. This helps to make important links stand out.# Set to '' if not wanted, or '...' is quite a good one.pagelinkicon='... <img src="http://pelican.cl.cam.ac.uk/icons/page.xbm"> ' # ---------------------------------------------------------------------# Less important personal preferences: # Sublistsize controls the maximum number of items the will appear as# an indented sub-list before the whole thing is moved onto a different# page. The smaller this is, the more pages you will have, but the# shorter each will be.sublistsize = 4 # That should be all.                                                 ######################################################################## for dir in os.curdir, os.environ['HOME']:    rcfile = os.path.join(dir, '.newslistrc.py')    if os.path.exists(rcfile):        print rcfile        execfile(rcfile)        break from nntplib import NNTPfrom stat import * rcsrev = '$Revision: 37320 $'rcsrev = string.join(filter(lambda s: '$' not in s, string.split(rcsrev)))desc = {} # Make (possibly) relative filenames into absolute onestreefile = os.path.join(topdir,treefile)descfile = os.path.join(topdir,descfile)page = os.path.join(topdir,pagedir) # First the bits for creating trees --------------------------- # Addtotree creates/augments a tree from a list of group namesdef addtotree(tree, groups):    print 'Updating tree...'    for i in groups:        parts = string.splitfields(i,'.')        makeleaf(tree, parts) # Makeleaf makes a leaf and the branch leading to it if necessarydef makeleaf(tree,path):    j = path[0]    l = len(path)     if not tree.has_key(j):        tree[j] = {}    if l == 1:        tree[j]['.'] = '.'    if l > 1:        makeleaf(tree[j],path[1:]) # Then the bits for outputting trees as pages ---------------- # Createpage creates an HTML file named <root>.html containing links# to those groups beginning with <root>. def createpage(root, tree, p):    filename = os.path.join(pagedir,root+'.html')    if root == rootpage:        detail = ''    else:        detail = ' under ' + root    f = open(filename,'w')    # f.write('Content-Type: text/html\n')    f.write('<TITLE>Newsgroups available' + detail + '</TITLE>\n')    f.write('<H1>Newsgroups available' + detail +'</H1>\n')    f.write('<A HREF="'+httppref+rootpage+'.html">Back to top level</A><P>\n')    printtree(f,tree,0,p)    f.write('<I>This page automatically created by \'newslist\' v. '+rcsrev+'.')    f.write(time.ctime(time.time()) + '</I><P>')    f.close() # Printtree prints the groups as a bulleted list.  Groups with# more than <sublistsize> subgroups will be put on a separate page.# Other sets of subgroups are just indented. def printtree(f, tree, indent, p):    global desc    l = len(tree)     if l > sublistsize and indent>0:        # Create a new page and a link to it        f.write('<LI><B><A HREF="'+httppref+p[1:]+'.html">')        f.write(p[1:]+'.*')        f.write('</A></B>'+pagelinkicon+'\n')        createpage(p[1:], tree, p)        return     kl = tree.keys()     if l > 1:        kl.sort()        if indent > 0:            # Create a sub-list            f.write('<LI>'+p[1:]+'\n<UL>')        else:            # Create a main list            f.write('<UL>')        indent = indent + 1     for i in kl:        if i == '.':            # Output a newsgroup            f.write('<LI><A HREF="news:' + p[1:] + '">'+ p[1:] + '</A> ')            if desc.has_key(p[1:]):                f.write('     <I>'+desc[p[1:]]+'</I>\n')            else:                f.write('\n')        else:            # Output a hierarchy            printtree(f,tree[i], indent, p+'.'+i)     if l > 1:        f.write('\n</UL>') # Reading descriptions file --------------------------------------- # This returns an array mapping group name to its description def readdesc(descfile):    global desc     desc = {}     if descfile == '':        return     try:        d = open(descfile, 'r')        print 'Reading descriptions...'    except (IOError):        print 'Failed to open description file ' + descfile        return    l = d.readline()    while l != '':        bits = string.split(l)        try:            grp = bits[0]            dsc = string.join(bits[1:])            if len(dsc)>1:                desc[grp] = dsc        except (IndexError):            pass        l = d.readline() # Check that ouput directory exists, ------------------------------# and offer to create it if not def checkopdir(pagedir):    if not os.path.isdir(pagedir):        print 'Directory '+pagedir+' does not exist.'        print 'Shall I create it for you? (y/n)'        if sys.stdin.readline()[0] == 'y':            try:                os.mkdir(pagedir,0777)            except:                print 'Sorry - failed!'                sys.exit(1)        else:            print 'OK. Exiting.'            sys.exit(1) # Read and write current local tree ---------------------------------- def readlocallist(treefile):    print 'Reading current local group list...'    tree = {}    try:        treetime = time.localtime(os.stat(treefile)[ST_MTIME])    except:        print '\n*** Failed to open local group cache '+treefile        print 'If this is the first time you have run newslist, then'        print 'use the -a option to create it.'        sys.exit(1)    treedate = '%02d%02d%02d' % (treetime[0] % 100 ,treetime[1], treetime[2])    try:        dump = open(treefile,'r')        tree = marshal.load(dump)        dump.close()    except (IOError):        print 'Cannot open local group list ' + treefile    return (tree, treedate) def writelocallist(treefile, tree):    try:        dump = open(treefile,'w')        groups = marshal.dump(tree,dump)        dump.close()        print 'Saved list to '+treefile+'\n'    except:        print 'Sorry - failed to write to local group cache '+treefile        print 'Does it (or its directory) have the correct permissions?'        sys.exit(1) # Return list of all groups on server ----------------------------- def getallgroups(server):    print 'Getting list of all groups...'    treedate='010101'    info = server.list()[1]    groups = []    print 'Processing...'    if skipempty:        print '\nIgnoring following empty groups:'    for i in info:        grpname = string.split(i[0])[0]        if skipempty and string.atoi(i[1]) < string.atoi(i[2]):            print grpname+' ',        else:            groups.append(grpname)    print '\n'    if skipempty:        print '(End of empty groups)'    return groups # Return list of new groups on server ----------------------------- def getnewgroups(server, treedate):    print 'Getting list of new groups since start of '+treedate+'...',    info = server.newgroups(treedate,'000001')[1]    print 'got %d.' % len(info)    print 'Processing...',    groups = []    for i in info:        grpname = string.split(i)[0]        groups.append(grpname)    print 'Done'    return groups # Now the main program -------------------------------------------- def main():    global desc     tree={}     # Check that the output directory exists    checkopdir(pagedir);     try:        print 'Connecting to '+newshost+'...'        if sys.version[0] == '0':            s = NNTP.init(newshost)        else:            s = NNTP(newshost)        connected = 1    except (nntplib.error_temp, nntplib.error_perm), x:        print 'Error connecting to host:', x        print 'I\'ll try to use just the local list.'        connected = 0     # If -a is specified, read the full list of groups from server    if connected and len(sys.argv) > 1 and sys.argv[1] == '-a':         groups = getallgroups(s)     # Otherwise just read the local file and then add    # groups created since local file last modified.    else:         (tree, treedate) = readlocallist(treefile)        if connected:            groups = getnewgroups(s, treedate)     if connected:        addtotree(tree, groups)        writelocallist(treefile,tree)     # Read group descriptions    readdesc(descfile)     print 'Creating pages...'    createpage(rootpage, tree, '')    print 'Done' if __name__ == "__main__":    main() # That's all folks######################################################################