lael-labs/9front

sys/src/cmd/python/Tools/scripts/README+66−0

@@ -0,0 +1,66 @@
	1	This directory contains a collection of executable Python scripts that
	2	are useful while building, extending or managing Python. Some (e.g.,
	3	dutree or lll) are also generally useful UNIX tools.
	4
	5	See also the Demo/scripts directory!
	6
	7	byext.py Print lines/words/chars stats of files by extension
	8	byteyears.py Print product of a file's size and age
	9	checkappend.py Search for multi-argument .append() calls
	10	checkpyc.py Check presence and validity of ".pyc" files
	11	classfix.py Convert old class syntax to new
	12	cleanfuture.py Fix reduntant Python __future__ statements
	13	combinerefs.py A helper for analyzing PYTHONDUMPREFS output.
	14	copytime.py Copy one file's atime and mtime to another
	15	crlf.py Change CRLF line endings to LF (Windows to Unix)
	16	cvsfiles.py Print a list of files that are under CVS
	17	db2pickle.py Dump a database file to a pickle
	18	diff.py Print file diffs in context, unified, or ndiff formats
	19	dutree.py Format du(1) output as a tree sorted by size
	20	eptags.py Create Emacs TAGS file for Python modules
	21	finddiv.py A grep-like tool that looks for division operators.
	22	findlinksto.py Recursively find symbolic links to a given path prefix
	23	findnocoding.py Find source files which need an encoding declaration
	24	fixcid.py Massive identifier substitution on C source files
	25	fixdiv.py Tool to fix division operators.
	26	fixheader.py Add some cpp magic to a C include file
	27	fixnotice.py Fix the copyright notice in source files
	28	fixps.py Fix Python scripts' first line (if #!)
	29	ftpmirror.py FTP mirror script
	30	google.py Open a webbrowser with Google.
	31	gprof2html.py Transform gprof(1) output into useful HTML.
	32	h2py.py Translate #define's into Python assignments
	33	hotshotmain.py Main program to run script under control of hotshot
	34	idle Main program to start IDLE
	35	ifdef.py Remove #if(n)def groups from C sources
	36	lfcr.py Change LF line endings to CRLF (Unix to Windows)
	37	linktree.py Make a copy of a tree with links to original files
	38	lll.py Find and list symbolic links in current directory
	39	logmerge.py Consolidate CVS/RCS logs read from stdin
	40	mailerdaemon.py parse error messages from mailer daemons (Sjoerd&Jack)
	41	md5sum.py Print MD5 checksums of argument files.
	42	methfix.py Fix old method syntax def f(self, (a1, ..., aN)):
	43	mkreal.py Turn a symbolic link into a real file or directory
	44	ndiff.py Intelligent diff between text files (Tim Peters)
	45	nm2def.py Create a template for PC/python_nt.def (Marc Lemburg)
	46	objgraph.py Print object graph from nm output on a library
	47	parseentities.py Utility for parsing HTML entity definitions
	48	pathfix.py Change #!/usr/local/bin/python into something else
	49	pdeps.py Print dependencies between Python modules
	50	pickle2db.py Load a pickle generated by db2pickle.py to a database
	51	pindent.py Indent Python code, giving block-closing comments
	52	ptags.py Create vi tags file for Python modules
	53	pydoc Python documentation browser.
	54	pysource.py Find Python source files
	55	redemo.py Basic regular expression demonstration facility
	56	reindent.py Change .py files to use 4-space indents.
	57	rgrep.py Reverse grep through a file (useful for big logfiles)
	58	setup.py Install all scripts listed here.
	59	suff.py Sort a list of files by suffix
	60	svneol.py Sets svn:eol-style on all files in directory.
	61	texcheck.py Validate Python LaTeX formatting (Raymond Hettinger)
	62	texi2html.py Convert GNU texinfo files into HTML
	63	treesync.py Synchronize source trees (very ideosyncratic)
	64	untabify.py Replace tabs with spaces in argument files
	65	which.py Find a program in $PATH
	66	xxci.py Wrapper for rcsdiff and ci

sys/src/cmd/python/Tools/scripts/byext.py+131−0

@@ -0,0 +1,131 @@
	1	#! /usr/bin/env python
	2
	3	"""Show file statistics by extension."""
	4
	5	import os
	6	import sys
	7
	8	class Stats:
	9
	10	def __init__(self):
	11	self.stats = {}
	12
	13	def statargs(self, args):
	14	for arg in args:
	15	if os.path.isdir(arg):
	16	self.statdir(arg)
	17	elif os.path.isfile(arg):
	18	self.statfile(arg)
	19	else:
	20	sys.stderr.write("Can't find %s\n" % arg)
	21	self.addstats("<???>", "unknown", 1)
	22
	23	def statdir(self, dir):
	24	self.addstats("<dir>", "dirs", 1)
	25	try:
	26	names = os.listdir(dir)
	27	except os.error, err:
	28	sys.stderr.write("Can't list %s: %s\n" % (dir, err))
	29	self.addstats("<dir>", "unlistable", 1)
	30	return
	31	names.sort()
	32	for name in names:
	33	if name.startswith(".#"):
	34	continue # Skip CVS temp files
	35	if name.endswith("~"):
	36	continue# Skip Emacs backup files
	37	full = os.path.join(dir, name)
	38	if os.path.islink(full):
	39	self.addstats("<lnk>", "links", 1)
	40	elif os.path.isdir(full):
	41	self.statdir(full)
	42	else:
	43	self.statfile(full)
	44
	45	def statfile(self, filename):
	46	head, ext = os.path.splitext(filename)
	47	head, base = os.path.split(filename)
	48	if ext == base:
	49	ext = "" # E.g. .cvsignore is deemed not to have an extension
	50	ext = os.path.normcase(ext)
	51	if not ext:
	52	ext = "<none>"
	53	self.addstats(ext, "files", 1)
	54	try:
	55	f = open(filename, "rb")
	56	except IOError, err:
	57	sys.stderr.write("Can't open %s: %s\n" % (filename, err))
	58	self.addstats(ext, "unopenable", 1)
	59	return
	60	data = f.read()
	61	f.close()
	62	self.addstats(ext, "bytes", len(data))
	63	if '\0' in data:
	64	self.addstats(ext, "binary", 1)
	65	return
	66	if not data:
	67	self.addstats(ext, "empty", 1)
	68	#self.addstats(ext, "chars", len(data))
	69	lines = data.splitlines()
	70	self.addstats(ext, "lines", len(lines))
	71	del lines
	72	words = data.split()
	73	self.addstats(ext, "words", len(words))
	74
	75	def addstats(self, ext, key, n):
	76	d = self.stats.setdefault(ext, {})
	77	d[key] = d.get(key, 0) + n
	78
	79	def report(self):
	80	exts = self.stats.keys()
	81	exts.sort()
	82	# Get the column keys
	83	columns = {}
	84	for ext in exts:
	85	columns.update(self.stats[ext])
	86	cols = columns.keys()
	87	cols.sort()
	88	colwidth = {}
	89	colwidth["ext"] = max([len(ext) for ext in exts])
	90	minwidth = 6
	91	self.stats["TOTAL"] = {}
	92	for col in cols:
	93	total = 0
	94	cw = max(minwidth, len(col))
	95	for ext in exts:
	96	value = self.stats[ext].get(col)
	97	if value is None:
	98	w = 0
	99	else:
	100	w = len("%d" % value)
	101	total += value
	102	cw = max(cw, w)
	103	cw = max(cw, len(str(total)))
	104	colwidth[col] = cw
	105	self.stats["TOTAL"][col] = total
	106	exts.append("TOTAL")
	107	for ext in exts:
	108	self.stats[ext]["ext"] = ext
	109	cols.insert(0, "ext")
	110	def printheader():
	111	for col in cols:
	112	print "%*s" % (colwidth[col], col),
	113	print
	114	printheader()
	115	for ext in exts:
	116	for col in cols:
	117	value = self.stats[ext].get(col, "")
	118	print "%*s" % (colwidth[col], value),
	119	print
	120	printheader() # Another header at the bottom
	121
	122	def main():
	123	args = sys.argv[1:]
	124	if not args:
	125	args = [os.curdir]
	126	s = Stats()
	127	s.statargs(args)
	128	s.report()
	129
	130	if __name__ == "__main__":
	131	main()

sys/src/cmd/python/Tools/scripts/byteyears.py+61−0

@@ -0,0 +1,61 @@
	1	#! /usr/bin/env python
	2
	3	# Print the product of age and size of each file, in suitable units.
	4	#
	5	# Usage: byteyears [ -a \| -m \| -c ] file ...
	6	#
	7	# Options -[amc] select atime, mtime (default) or ctime as age.
	8
	9	import sys, os, time
	10	from stat import *
	11
	12	def main():
	13
	14	# Use lstat() to stat files if it exists, else stat()
	15	try:
	16	statfunc = os.lstat
	17	except AttributeError:
	18	statfunc = os.stat
	19
	20	# Parse options
	21	if sys.argv[1] == '-m':
	22	itime = ST_MTIME
	23	del sys.argv[1]
	24	elif sys.argv[1] == '-c':
	25	itime = ST_CTIME
	26	del sys.argv[1]
	27	elif sys.argv[1] == '-a':
	28	itime = ST_CTIME
	29	del sys.argv[1]
	30	else:
	31	itime = ST_MTIME
	32
	33	secs_per_year = 365.0 * 24.0 * 3600.0 # Scale factor
	34	now = time.time() # Current time, for age computations
	35	status = 0 # Exit status, set to 1 on errors
	36
	37	# Compute max file name length
	38	maxlen = 1
	39	for filename in sys.argv[1:]:
	40	maxlen = max(maxlen, len(filename))
	41
	42	# Process each argument in turn
	43	for filename in sys.argv[1:]:
	44	try:
	45	st = statfunc(filename)
	46	except os.error, msg:
	47	sys.stderr.write("can't stat %r: %r\n" % (filename, msg))
	48	status = 1
	49	st = ()
	50	if st:
	51	anytime = st[itime]
	52	size = st[ST_SIZE]
	53	age = now - anytime
	54	byteyears = float(size) * float(age) / secs_per_year
	55	print filename.ljust(maxlen),
	56	print repr(int(byteyears)).rjust(8)
	57
	58	sys.exit(status)
	59
	60	if __name__ == '__main__':
	61	main()

sys/src/cmd/python/Tools/scripts/checkappend.py+167−0

@@ -0,0 +1,167 @@
	1	#! /usr/bin/env python
	2
	3	# Released to the public domain, by Tim Peters, 28 February 2000.
	4
	5	"""checkappend.py -- search for multi-argument .append() calls.
	6
	7	Usage: specify one or more file or directory paths:
	8	checkappend [-v] file_or_dir [file_or_dir] ...
	9
	10	Each file_or_dir is checked for multi-argument .append() calls. When
	11	a directory, all .py files in the directory, and recursively in its
	12	subdirectories, are checked.
	13
	14	Use -v for status msgs. Use -vv for more status msgs.
	15
	16	In the absence of -v, the only output is pairs of the form
	17
	18	filename(linenumber):
	19	line containing the suspicious append
	20
	21	Note that this finds multi-argument append calls regardless of whether
	22	they're attached to list objects. If a module defines a class with an
	23	append method that takes more than one argument, calls to that method
	24	will be listed.
	25
	26	Note that this will not find multi-argument list.append calls made via a
	27	bound method object. For example, this is not caught:
	28
	29	somelist = []
	30	push = somelist.append
	31	push(1, 2, 3)
	32	"""
	33
	34	__version__ = 1, 0, 0
	35
	36	import os
	37	import sys
	38	import getopt
	39	import tokenize
	40
	41	verbose = 0
	42
	43	def errprint(*args):
	44	msg = ' '.join(args)
	45	sys.stderr.write(msg)
	46	sys.stderr.write("\n")
	47
	48	def main():
	49	args = sys.argv[1:]
	50	global verbose
	51	try:
	52	opts, args = getopt.getopt(sys.argv[1:], "v")
	53	except getopt.error, msg:
	54	errprint(str(msg) + "\n\n" + __doc__)
	55	return
	56	for opt, optarg in opts:
	57	if opt == '-v':
	58	verbose = verbose + 1
	59	if not args:
	60	errprint(__doc__)
	61	return
	62	for arg in args:
	63	check(arg)
	64
	65	def check(file):
	66	if os.path.isdir(file) and not os.path.islink(file):
	67	if verbose:
	68	print "%r: listing directory" % (file,)
	69	names = os.listdir(file)
	70	for name in names:
	71	fullname = os.path.join(file, name)
	72	if ((os.path.isdir(fullname) and
	73	not os.path.islink(fullname))
	74	or os.path.normcase(name[-3:]) == ".py"):
	75	check(fullname)
	76	return
	77
	78	try:
	79	f = open(file)
	80	except IOError, msg:
	81	errprint("%r: I/O Error: %s" % (file, msg))
	82	return
	83
	84	if verbose > 1:
	85	print "checking %r ..." % (file,)
	86
	87	ok = AppendChecker(file, f).run()
	88	if verbose and ok:
	89	print "%r: Clean bill of health." % (file,)
	90
	91	[FIND_DOT,
	92	FIND_APPEND,
	93	FIND_LPAREN,
	94	FIND_COMMA,
	95	FIND_STMT] = range(5)
	96
	97	class AppendChecker:
	98	def __init__(self, fname, file):
	99	self.fname = fname
	100	self.file = file
	101	self.state = FIND_DOT
	102	self.nerrors = 0
	103
	104	def run(self):
	105	try:
	106	tokenize.tokenize(self.file.readline, self.tokeneater)
	107	except tokenize.TokenError, msg:
	108	errprint("%r: Token Error: %s" % (self.fname, msg))
	109	self.nerrors = self.nerrors + 1
	110	return self.nerrors == 0
	111
	112	def tokeneater(self, type, token, start, end, line,
	113	NEWLINE=tokenize.NEWLINE,
	114	JUNK=(tokenize.COMMENT, tokenize.NL),
	115	OP=tokenize.OP,
	116	NAME=tokenize.NAME):
	117
	118	state = self.state
	119
	120	if type in JUNK:
	121	pass
	122
	123	elif state is FIND_DOT:
	124	if type is OP and token == ".":
	125	state = FIND_APPEND
	126
	127	elif state is FIND_APPEND:
	128	if type is NAME and token == "append":
	129	self.line = line
	130	self.lineno = start[0]
	131	state = FIND_LPAREN
	132	else:
	133	state = FIND_DOT
	134
	135	elif state is FIND_LPAREN:
	136	if type is OP and token == "(":
	137	self.level = 1
	138	state = FIND_COMMA
	139	else:
	140	state = FIND_DOT
	141
	142	elif state is FIND_COMMA:
	143	if type is OP:
	144	if token in ("(", "{", "["):
	145	self.level = self.level + 1
	146	elif token in (")", "}", "]"):
	147	self.level = self.level - 1
	148	if self.level == 0:
	149	state = FIND_DOT
	150	elif token == "," and self.level == 1:
	151	self.nerrors = self.nerrors + 1
	152	print "%s(%d):\n%s" % (self.fname, self.lineno,
	153	self.line)
	154	# don't gripe about this stmt again
	155	state = FIND_STMT
	156
	157	elif state is FIND_STMT:
	158	if type is NEWLINE:
	159	state = FIND_DOT
	160
	161	else:
	162	raise SystemError("unknown internal state '%r'" % (state,))
	163
	164	self.state = state
	165
	166	if __name__ == '__main__':
	167	main()

sys/src/cmd/python/Tools/scripts/checkpyc.py+66−0

@@ -0,0 +1,66 @@
	1	#! /usr/bin/env python
	2	# Check that all ".pyc" files exist and are up-to-date
	3	# Uses module 'os'
	4
	5	import sys
	6	import os
	7	from stat import ST_MTIME
	8	import imp
	9
	10	def main():
	11	silent = 0
	12	verbose = 0
	13	if sys.argv[1:]:
	14	if sys.argv[1] == '-v':
	15	verbose = 1
	16	elif sys.argv[1] == '-s':
	17	silent = 1
	18	MAGIC = imp.get_magic()
	19	if not silent:
	20	print 'Using MAGIC word', repr(MAGIC)
	21	for dirname in sys.path:
	22	try:
	23	names = os.listdir(dirname)
	24	except os.error:
	25	print 'Cannot list directory', repr(dirname)
	26	continue
	27	if not silent:
	28	print 'Checking ', repr(dirname), '...'
	29	names.sort()
	30	for name in names:
	31	if name[-3:] == '.py':
	32	name = os.path.join(dirname, name)
	33	try:
	34	st = os.stat(name)
	35	except os.error:
	36	print 'Cannot stat', repr(name)
	37	continue
	38	if verbose:
	39	print 'Check', repr(name), '...'
	40	name_c = name + 'c'
	41	try:
	42	f = open(name_c, 'r')
	43	except IOError:
	44	print 'Cannot open', repr(name_c)
	45	continue
	46	magic_str = f.read(4)
	47	mtime_str = f.read(4)
	48	f.close()
	49	if magic_str <> MAGIC:
	50	print 'Bad MAGIC word in ".pyc" file',
	51	print repr(name_c)
	52	continue
	53	mtime = get_long(mtime_str)
	54	if mtime == 0 or mtime == -1:
	55	print 'Bad ".pyc" file', repr(name_c)
	56	elif mtime <> st[ST_MTIME]:
	57	print 'Out-of-date ".pyc" file',
	58	print repr(name_c)
	59
	60	def get_long(s):
	61	if len(s) <> 4:
	62	return -1
	63	return ord(s[0]) + (ord(s[1])<<8) + (ord(s[2])<<16) + (ord(s[3])<<24)
	64
	65	if __name__ == '__main__':
	66	main()

sys/src/cmd/python/Tools/scripts/classfix.py+190−0

@@ -0,0 +1,190 @@
	1	#! /usr/bin/env python
	2
	3	# This script is obsolete -- it is kept for historical purposes only.
	4	#
	5	# Fix Python source files to use the new class definition syntax, i.e.,
	6	# the syntax used in Python versions before 0.9.8:
	7	# class C() = base(), base(), ...: ...
	8	# is changed to the current syntax:
	9	# class C(base, base, ...): ...
	10	#
	11	# The script uses heuristics to find class definitions that usually
	12	# work but occasionally can fail; carefully check the output!
	13	#
	14	# Command line arguments are files or directories to be processed.
	15	# Directories are searched recursively for files whose name looks
	16	# like a python module.
	17	# Symbolic links are always ignored (except as explicit directory
	18	# arguments). Of course, the original file is kept as a back-up
	19	# (with a "~" attached to its name).
	20	#
	21	# Changes made are reported to stdout in a diff-like format.
	22	#
	23	# Undoubtedly you can do this using find and sed or perl, but this is
	24	# a nice example of Python code that recurses down a directory tree
	25	# and uses regular expressions. Also note several subtleties like
	26	# preserving the file's mode and avoiding to even write a temp file
	27	# when no changes are needed for a file.
	28	#
	29	# NB: by changing only the function fixline() you can turn this
	30	# into a program for a different change to Python programs...
	31
	32	import sys
	33	import re
	34	import os
	35	from stat import *
	36
	37	err = sys.stderr.write
	38	dbg = err
	39	rep = sys.stdout.write
	40
	41	def main():
	42	bad = 0
	43	if not sys.argv[1:]: # No arguments
	44	err('usage: ' + sys.argv[0] + ' file-or-directory ...\n')
	45	sys.exit(2)
	46	for arg in sys.argv[1:]:
	47	if os.path.isdir(arg):
	48	if recursedown(arg): bad = 1
	49	elif os.path.islink(arg):
	50	err(arg + ': will not process symbolic links\n')
	51	bad = 1
	52	else:
	53	if fix(arg): bad = 1
	54	sys.exit(bad)
	55
	56	ispythonprog = re.compile('^[a-zA-Z0-9_]+\.py$')
	57	def ispython(name):
	58	return ispythonprog.match(name) >= 0
	59
	60	def recursedown(dirname):
	61	dbg('recursedown(%r)\n' % (dirname,))
	62	bad = 0
	63	try:
	64	names = os.listdir(dirname)
	65	except os.error, msg:
	66	err('%s: cannot list directory: %r\n' % (dirname, msg))
	67	return 1
	68	names.sort()
	69	subdirs = []
	70	for name in names:
	71	if name in (os.curdir, os.pardir): continue
	72	fullname = os.path.join(dirname, name)
	73	if os.path.islink(fullname): pass
	74	elif os.path.isdir(fullname):
	75	subdirs.append(fullname)
	76	elif ispython(name):
	77	if fix(fullname): bad = 1
	78	for fullname in subdirs:
	79	if recursedown(fullname): bad = 1
	80	return bad
	81
	82	def fix(filename):
	83	## dbg('fix(%r)\n' % (filename,))
	84	try:
	85	f = open(filename, 'r')
	86	except IOError, msg:
	87	err('%s: cannot open: %r\n' % (filename, msg))
	88	return 1
	89	head, tail = os.path.split(filename)
	90	tempname = os.path.join(head, '@' + tail)
	91	g = None
	92	# If we find a match, we rewind the file and start over but
	93	# now copy everything to a temp file.
	94	lineno = 0
	95	while 1:
	96	line = f.readline()
	97	if not line: break
	98	lineno = lineno + 1
	99	while line[-2:] == '\\\n':
	100	nextline = f.readline()
	101	if not nextline: break
	102	line = line + nextline
	103	lineno = lineno + 1
	104	newline = fixline(line)
	105	if newline != line:
	106	if g is None:
	107	try:
	108	g = open(tempname, 'w')
	109	except IOError, msg:
	110	f.close()
	111	err('%s: cannot create: %r\n' % (tempname, msg))
	112	return 1
	113	f.seek(0)
	114	lineno = 0
	115	rep(filename + ':\n')
	116	continue # restart from the beginning
	117	rep(repr(lineno) + '\n')
	118	rep('< ' + line)
	119	rep('> ' + newline)
	120	if g is not None:
	121	g.write(newline)
	122
	123	# End of file
	124	f.close()
	125	if not g: return 0 # No changes
	126
	127	# Finishing touch -- move files
	128
	129	# First copy the file's mode to the temp file
	130	try:
	131	statbuf = os.stat(filename)
	132	os.chmod(tempname, statbuf[ST_MODE] & 07777)
	133	except os.error, msg:
	134	err('%s: warning: chmod failed (%r)\n' % (tempname, msg))
	135	# Then make a backup of the original file as filename~
	136	try:
	137	os.rename(filename, filename + '~')
	138	except os.error, msg:
	139	err('%s: warning: backup failed (%r)\n' % (filename, msg))
	140	# Now move the temp file to the original file
	141	try:
	142	os.rename(tempname, filename)
	143	except os.error, msg:
	144	err('%s: rename failed (%r)\n' % (filename, msg))
	145	return 1
	146	# Return succes
	147	return 0
	148
	149	# This expression doesn't catch all class definition headers,
	150	# but it's pretty darn close.
	151	classexpr = '^([ \t]class +[a-zA-Z0-9_]+) ( ) ((=.*)?):'
	152	classprog = re.compile(classexpr)
	153
	154	# Expressions for finding base class expressions.
	155	baseexpr = '^ (.) ( ) *$'
	156	baseprog = re.compile(baseexpr)
	157
	158	def fixline(line):
	159	if classprog.match(line) < 0: # No 'class' keyword -- no change
	160	return line
	161
	162	(a0, b0), (a1, b1), (a2, b2) = classprog.regs[:3]
	163	# a0, b0 = Whole match (up to ':')
	164	# a1, b1 = First subexpression (up to classname)
	165	# a2, b2 = Second subexpression (=.*)
	166	head = line[:b1]
	167	tail = line[b0:] # Unmatched rest of line
	168
	169	if a2 == b2: # No base classes -- easy case
	170	return head + ':' + tail
	171
	172	# Get rid of leading '='
	173	basepart = line[a2+1:b2]
	174
	175	# Extract list of base expressions
	176	bases = basepart.split(',')
	177
	178	# Strip trailing '()' from each base expression
	179	for i in range(len(bases)):
	180	if baseprog.match(bases[i]) >= 0:
	181	x1, y1 = baseprog.regs[1]
	182	bases[i] = bases[i][x1:y1]
	183
	184	# Join the bases back again and build the new line
	185	basepart = ', '.join(bases)
	186
	187	return head + '(' + basepart + '):' + tail
	188
	189	if __name__ == '__main__':
	190	main()

sys/src/cmd/python/Tools/scripts/cleanfuture.py+276−0

@@ -0,0 +1,276 @@
	1	#! /usr/bin/env python
	2
	3	"""cleanfuture [-d][-r][-v] path ...
	4
	5	-d Dry run. Analyze, but don't make any changes to, files.
	6	-r Recurse. Search for all .py files in subdirectories too.
	7	-v Verbose. Print informative msgs.
	8
	9	Search Python (.py) files for future statements, and remove the features
	10	from such statements that are already mandatory in the version of Python
	11	you're using.
	12
	13	Pass one or more file and/or directory paths. When a directory path, all
	14	.py files within the directory will be examined, and, if the -r option is
	15	given, likewise recursively for subdirectories.
	16
	17	Overwrites files in place, renaming the originals with a .bak extension. If
	18	cleanfuture finds nothing to change, the file is left alone. If cleanfuture
	19	does change a file, the changed file is a fixed-point (i.e., running
	20	cleanfuture on the resulting .py file won't change it again, at least not
	21	until you try it again with a later Python release).
	22
	23	Limitations: You can do these things, but this tool won't help you then:
	24
	25	+ A future statement cannot be mixed with any other statement on the same
	26	physical line (separated by semicolon).
	27
	28	+ A future statement cannot contain an "as" clause.
	29
	30	Example: Assuming you're using Python 2.2, if a file containing
	31
	32	from __future__ import nested_scopes, generators
	33
	34	is analyzed by cleanfuture, the line is rewritten to
	35
	36	from __future__ import generators
	37
	38	because nested_scopes is no longer optional in 2.2 but generators is.
	39	"""
	40
	41	import __future__
	42	import tokenize
	43	import os
	44	import sys
	45
	46	dryrun = 0
	47	recurse = 0
	48	verbose = 0
	49
	50	def errprint(*args):
	51	strings = map(str, args)
	52	msg = ' '.join(strings)
	53	if msg[-1:] != '\n':
	54	msg += '\n'
	55	sys.stderr.write(msg)
	56
	57	def main():
	58	import getopt
	59	global verbose, recurse, dryrun
	60	try:
	61	opts, args = getopt.getopt(sys.argv[1:], "drv")
	62	except getopt.error, msg:
	63	errprint(msg)
	64	return
	65	for o, a in opts:
	66	if o == '-d':
	67	dryrun += 1
	68	elif o == '-r':
	69	recurse += 1
	70	elif o == '-v':
	71	verbose += 1
	72	if not args:
	73	errprint("Usage:", __doc__)
	74	return
	75	for arg in args:
	76	check(arg)
	77
	78	def check(file):
	79	if os.path.isdir(file) and not os.path.islink(file):
	80	if verbose:
	81	print "listing directory", file
	82	names = os.listdir(file)
	83	for name in names:
	84	fullname = os.path.join(file, name)
	85	if ((recurse and os.path.isdir(fullname) and
	86	not os.path.islink(fullname))
	87	or name.lower().endswith(".py")):
	88	check(fullname)
	89	return
	90
	91	if verbose:
	92	print "checking", file, "...",
	93	try:
	94	f = open(file)
	95	except IOError, msg:
	96	errprint("%r: I/O Error: %s" % (file, str(msg)))
	97	return
	98
	99	ff = FutureFinder(f, file)
	100	changed = ff.run()
	101	if changed:
	102	ff.gettherest()
	103	f.close()
	104	if changed:
	105	if verbose:
	106	print "changed."
	107	if dryrun:
	108	print "But this is a dry run, so leaving it alone."
	109	for s, e, line in changed:
	110	print "%r lines %d-%d" % (file, s+1, e+1)
	111	for i in range(s, e+1):
	112	print ff.lines[i],
	113	if line is None:
	114	print "-- deleted"
	115	else:
	116	print "-- change to:"
	117	print line,
	118	if not dryrun:
	119	bak = file + ".bak"
	120	if os.path.exists(bak):
	121	os.remove(bak)
	122	os.rename(file, bak)
	123	if verbose:
	124	print "renamed", file, "to", bak
	125	g = open(file, "w")
	126	ff.write(g)
	127	g.close()
	128	if verbose:
	129	print "wrote new", file
	130	else:
	131	if verbose:
	132	print "unchanged."
	133
	134	class FutureFinder:
	135
	136	def __init__(self, f, fname):
	137	self.f = f
	138	self.fname = fname
	139	self.ateof = 0
	140	self.lines = [] # raw file lines
	141
	142	# List of (start_index, end_index, new_line) triples.
	143	self.changed = []
	144
	145	# Line-getter for tokenize.
	146	def getline(self):
	147	if self.ateof:
	148	return ""
	149	line = self.f.readline()
	150	if line == "":
	151	self.ateof = 1
	152	else:
	153	self.lines.append(line)
	154	return line
	155
	156	def run(self):
	157	STRING = tokenize.STRING
	158	NL = tokenize.NL
	159	NEWLINE = tokenize.NEWLINE
	160	COMMENT = tokenize.COMMENT
	161	NAME = tokenize.NAME
	162	OP = tokenize.OP
	163
	164	changed = self.changed
	165	get = tokenize.generate_tokens(self.getline).next
	166	type, token, (srow, scol), (erow, ecol), line = get()
	167
	168	# Chew up initial comments and blank lines (if any).
	169	while type in (COMMENT, NL, NEWLINE):
	170	type, token, (srow, scol), (erow, ecol), line = get()
	171
	172	# Chew up docstring (if any -- and it may be implicitly catenated!).
	173	while type is STRING:
	174	type, token, (srow, scol), (erow, ecol), line = get()
	175
	176	# Analyze the future stmts.
	177	while 1:
	178	# Chew up comments and blank lines (if any).
	179	while type in (COMMENT, NL, NEWLINE):
	180	type, token, (srow, scol), (erow, ecol), line = get()
	181
	182	if not (type is NAME and token == "from"):
	183	break
	184	startline = srow - 1 # tokenize is one-based
	185	type, token, (srow, scol), (erow, ecol), line = get()
	186
	187	if not (type is NAME and token == "__future__"):
	188	break
	189	type, token, (srow, scol), (erow, ecol), line = get()
	190
	191	if not (type is NAME and token == "import"):
	192	break
	193	type, token, (srow, scol), (erow, ecol), line = get()
	194
	195	# Get the list of features.
	196	features = []
	197	while type is NAME:
	198	features.append(token)
	199	type, token, (srow, scol), (erow, ecol), line = get()
	200
	201	if not (type is OP and token == ','):
	202	break
	203	type, token, (srow, scol), (erow, ecol), line = get()
	204
	205	# A trailing comment?
	206	comment = None
	207	if type is COMMENT:
	208	comment = token
	209	type, token, (srow, scol), (erow, ecol), line = get()
	210
	211	if type is not NEWLINE:
	212	errprint("Skipping file %r; can't parse line %d:\n%s" %
	213	(self.fname, srow, line))
	214	return []
	215
	216	endline = srow - 1
	217
	218	# Check for obsolete features.
	219	okfeatures = []
	220	for f in features:
	221	object = getattr(__future__, f, None)
	222	if object is None:
	223	# A feature we don't know about yet -- leave it in.
	224	# They'll get a compile-time error when they compile
	225	# this program, but that's not our job to sort out.
	226	okfeatures.append(f)
	227	else:
	228	released = object.getMandatoryRelease()
	229	if released is None or released <= sys.version_info:
	230	# Withdrawn or obsolete.
	231	pass
	232	else:
	233	okfeatures.append(f)
	234
	235	# Rewrite the line if at least one future-feature is obsolete.
	236	if len(okfeatures) < len(features):
	237	if len(okfeatures) == 0:
	238	line = None
	239	else:
	240	line = "from __future__ import "
	241	line += ', '.join(okfeatures)
	242	if comment is not None:
	243	line += ' ' + comment
	244	line += '\n'
	245	changed.append((startline, endline, line))
	246
	247	# Loop back for more future statements.
	248
	249	return changed
	250
	251	def gettherest(self):
	252	if self.ateof:
	253	self.therest = ''
	254	else:
	255	self.therest = self.f.read()
	256
	257	def write(self, f):
	258	changed = self.changed
	259	assert changed
	260	# Prevent calling this again.
	261	self.changed = []
	262	# Apply changes in reverse order.
	263	changed.reverse()
	264	for s, e, line in changed:
	265	if line is None:
	266	# pure deletion
	267	del self.lines[s:e+1]
	268	else:
	269	self.lines[s:e+1] = [line]
	270	f.writelines(self.lines)
	271	# Copy over the remainder of the file.
	272	if self.therest:
	273	f.write(self.therest)
	274
	275	if __name__ == '__main__':
	276	main()

sys/src/cmd/python/Tools/scripts/combinerefs.py+127−0

@@ -0,0 +1,127 @@
	1	#! /usr/bin/env python
	2
	3	"""
	4	combinerefs path
	5
	6	A helper for analyzing PYTHONDUMPREFS output.
	7
	8	When the PYTHONDUMPREFS envar is set in a debug build, at Python shutdown
	9	time Py_Finalize() prints the list of all live objects twice: first it
	10	prints the repr() of each object while the interpreter is still fully intact.
	11	After cleaning up everything it can, it prints all remaining live objects
	12	again, but the second time just prints their addresses, refcounts, and type
	13	names (because the interpreter has been torn down, calling repr methods at
	14	this point can get into infinite loops or blow up).
	15
	16	Save all this output into a file, then run this script passing the path to
	17	that file. The script finds both output chunks, combines them, then prints
	18	a line of output for each object still alive at the end:
	19
	20	address refcnt typename repr
	21
	22	address is the address of the object, in whatever format the platform C
	23	produces for a %p format code.
	24
	25	refcnt is of the form
	26
	27	"[" ref "]"
	28
	29	when the object's refcount is the same in both PYTHONDUMPREFS output blocks,
	30	or
	31
	32	"[" ref_before "->" ref_after "]"
	33
	34	if the refcount changed.
	35
	36	typename is object->ob_type->tp_name, extracted from the second PYTHONDUMPREFS
	37	output block.
	38
	39	repr is repr(object), extracted from the first PYTHONDUMPREFS output block.
	40	CAUTION: If object is a container type, it may not actually contain all the
	41	objects shown in the repr: the repr was captured from the first output block,
	42	and some of the containees may have been released since then. For example,
	43	it's common for the line showing the dict of interned strings to display
	44	strings that no longer exist at the end of Py_Finalize; this can be recognized
	45	(albeit painfully) because such containees don't have a line of their own.
	46
	47	The objects are listed in allocation order, with most-recently allocated
	48	printed first, and the first object allocated printed last.
	49
	50
	51	Simple examples:
	52
	53	00857060 [14] str '__len__'
	54
	55	The str object '__len__' is alive at shutdown time, and both PYTHONDUMPREFS
	56	output blocks said there were 14 references to it. This is probably due to
	57	C modules that intern the string "__len__" and keep a reference to it in a
	58	file static.
	59
	60	00857038 [46->5] tuple ()
	61
	62	46-5 = 41 references to the empty tuple were removed by the cleanup actions
	63	between the times PYTHONDUMPREFS produced output.
	64
	65	00858028 [1025->1456] str '<dummy key>'
	66
	67	The string '<dummy key>', which is used in dictobject.c to overwrite a real
	68	key that gets deleted, grew several hundred references during cleanup. It
	69	suggests that stuff did get removed from dicts by cleanup, but that the dicts
	70	themselves are staying alive for some reason. """
	71
	72	import re
	73	import sys
	74
	75	# Generate lines from fileiter. If whilematch is true, continue reading
	76	# while the regexp object pat matches line. If whilematch is false, lines
	77	# are read so long as pat doesn't match them. In any case, the first line
	78	# that doesn't match pat (when whilematch is true), or that does match pat
	79	# (when whilematch is false), is lost, and fileiter will resume at the line
	80	# following it.
	81	def read(fileiter, pat, whilematch):
	82	for line in fileiter:
	83	if bool(pat.match(line)) == whilematch:
	84	yield line
	85	else:
	86	break
	87
	88	def combine(fname):
	89	f = file(fname)
	90	fi = iter(f)
	91
	92	for line in read(fi, re.compile(r'^Remaining objects:$'), False):
	93	pass
	94
	95	crack = re.compile(r'([a-zA-Z\d]+) \[(\d+)\] (.*)')
	96	addr2rc = {}
	97	addr2guts = {}
	98	before = 0
	99	for line in read(fi, re.compile(r'^Remaining object addresses:$'), False):
	100	m = crack.match(line)
	101	if m:
	102	addr, addr2rc[addr], addr2guts[addr] = m.groups()
	103	before += 1
	104	else:
	105	print '??? skipped:', line
	106
	107	after = 0
	108	for line in read(fi, crack, True):
	109	after += 1
	110	m = crack.match(line)
	111	assert m
	112	addr, rc, guts = m.groups() # guts is type name here
	113	if addr not in addr2rc:
	114	print '??? new object created while tearing down:', line.rstrip()
	115	continue
	116	print addr,
	117	if rc == addr2rc[addr]:
	118	print '[%s]' % rc,
	119	else:
	120	print '[%s->%s]' % (addr2rc[addr], rc),
	121	print guts, addr2guts[addr]
	122
	123	f.close()
	124	print "%d objects before, %d after" % (before, after)
	125
	126	if __name__ == '__main__':
	127	combine(sys.argv[1])

sys/src/cmd/python/Tools/scripts/copytime.py+26−0

@@ -0,0 +1,26 @@
	1	#! /usr/bin/env python
	2
	3	# Copy one file's atime and mtime to another
	4
	5	import sys
	6	import os
	7	from stat import ST_ATIME, ST_MTIME # Really constants 7 and 8
	8
	9	def main():
	10	if len(sys.argv) <> 3:
	11	sys.stderr.write('usage: copytime source destination\n')
	12	sys.exit(2)
	13	file1, file2 = sys.argv[1], sys.argv[2]
	14	try:
	15	stat1 = os.stat(file1)
	16	except os.error:
	17	sys.stderr.write(file1 + ': cannot stat\n')
	18	sys.exit(1)
	19	try:
	20	os.utime(file2, (stat1[ST_ATIME], stat1[ST_MTIME]))
	21	except os.error:
	22	sys.stderr.write(file2 + ': cannot change time\n')
	23	sys.exit(2)
	24
	25	if __name__ == '__main__':
	26	main()

sys/src/cmd/python/Tools/scripts/crlf.py+23−0

@@ -0,0 +1,23 @@
	1	#! /usr/bin/env python
	2	"Replace CRLF with LF in argument files. Print names of changed files."
	3
	4	import sys, os
	5
	6	def main():
	7	for filename in sys.argv[1:]:
	8	if os.path.isdir(filename):
	9	print filename, "Directory!"
	10	continue
	11	data = open(filename, "rb").read()
	12	if '\0' in data:
	13	print filename, "Binary!"
	14	continue
	15	newdata = data.replace("\r\n", "\n")
	16	if newdata != data:
	17	print filename
	18	f = open(filename, "wb")
	19	f.write(newdata)
	20	f.close()
	21
	22	if __name__ == '__main__':
	23	main()

sys/src/cmd/python/Tools/scripts/cvsfiles.py+72−0

@@ -0,0 +1,72 @@
	1	#! /usr/bin/env python
	2
	3	"""Print a list of files that are mentioned in CVS directories.
	4
	5	Usage: cvsfiles.py [-n file] [directory] ...
	6
	7	If the '-n file' option is given, only files under CVS that are newer
	8	than the given file are printed; by default, all files under CVS are
	9	printed. As a special case, if a file does not exist, it is always
	10	printed.
	11	"""
	12
	13	import os
	14	import sys
	15	import stat
	16	import getopt
	17
	18	cutofftime = 0
	19
	20	def main():
	21	try:
	22	opts, args = getopt.getopt(sys.argv[1:], "n:")
	23	except getopt.error, msg:
	24	print msg
	25	print __doc__,
	26	return 1
	27	global cutofftime
	28	newerfile = None
	29	for o, a in opts:
	30	if o == '-n':
	31	cutofftime = getmtime(a)
	32	if args:
	33	for arg in args:
	34	process(arg)
	35	else:
	36	process(".")
	37
	38	def process(dir):
	39	cvsdir = 0
	40	subdirs = []
	41	names = os.listdir(dir)
	42	for name in names:
	43	fullname = os.path.join(dir, name)
	44	if name == "CVS":
	45	cvsdir = fullname
	46	else:
	47	if os.path.isdir(fullname):
	48	if not os.path.islink(fullname):
	49	subdirs.append(fullname)
	50	if cvsdir:
	51	entries = os.path.join(cvsdir, "Entries")
	52	for e in open(entries).readlines():
	53	words = e.split('/')
	54	if words[0] == '' and words[1:]:
	55	name = words[1]
	56	fullname = os.path.join(dir, name)
	57	if cutofftime and getmtime(fullname) <= cutofftime:
	58	pass
	59	else:
	60	print fullname
	61	for sub in subdirs:
	62	process(sub)
	63
	64	def getmtime(filename):
	65	try:
	66	st = os.stat(filename)
	67	except os.error:
	68	return 0
	69	return st[stat.ST_MTIME]
	70
	71	if __name__ == '__main__':
	72	main()

sys/src/cmd/python/Tools/scripts/db2pickle.py+135−0

@@ -0,0 +1,135 @@
	1	#!/usr/bin/env python
	2
	3	"""
	4	Synopsis: %(prog)s [-h\|-g\|-b\|-r\|-a] dbfile [ picklefile ]
	5
	6	Convert the database file given on the command line to a pickle
	7	representation. The optional flags indicate the type of the database:
	8
	9	-a - open using anydbm
	10	-b - open as bsddb btree file
	11	-d - open as dbm file
	12	-g - open as gdbm file
	13	-h - open as bsddb hash file
	14	-r - open as bsddb recno file
	15
	16	The default is hash. If a pickle file is named it is opened for write
	17	access (deleting any existing data). If no pickle file is named, the pickle
	18	output is written to standard output.
	19
	20	"""
	21
	22	import getopt
	23	try:
	24	import bsddb
	25	except ImportError:
	26	bsddb = None
	27	try:
	28	import dbm
	29	except ImportError:
	30	dbm = None
	31	try:
	32	import gdbm
	33	except ImportError:
	34	gdbm = None
	35	try:
	36	import anydbm
	37	except ImportError:
	38	anydbm = None
	39	import sys
	40	try:
	41	import cPickle as pickle
	42	except ImportError:
	43	import pickle
	44
	45	prog = sys.argv[0]
	46
	47	def usage():
	48	sys.stderr.write(__doc__ % globals())
	49
	50	def main(args):
	51	try:
	52	opts, args = getopt.getopt(args, "hbrdag",
	53	["hash", "btree", "recno", "dbm",
	54	"gdbm", "anydbm"])
	55	except getopt.error:
	56	usage()
	57	return 1
	58
	59	if len(args) == 0 or len(args) > 2:
	60	usage()
	61	return 1
	62	elif len(args) == 1:
	63	dbfile = args[0]
	64	pfile = sys.stdout
	65	else:
	66	dbfile = args[0]
	67	try:
	68	pfile = open(args[1], 'wb')
	69	except IOError:
	70	sys.stderr.write("Unable to open %s\n" % args[1])
	71	return 1
	72
	73	dbopen = None
	74	for opt, arg in opts:
	75	if opt in ("-h", "--hash"):
	76	try:
	77	dbopen = bsddb.hashopen
	78	except AttributeError:
	79	sys.stderr.write("bsddb module unavailable.\n")
	80	return 1
	81	elif opt in ("-b", "--btree"):
	82	try:
	83	dbopen = bsddb.btopen
	84	except AttributeError:
	85	sys.stderr.write("bsddb module unavailable.\n")
	86	return 1
	87	elif opt in ("-r", "--recno"):
	88	try:
	89	dbopen = bsddb.rnopen
	90	except AttributeError:
	91	sys.stderr.write("bsddb module unavailable.\n")
	92	return 1
	93	elif opt in ("-a", "--anydbm"):
	94	try:
	95	dbopen = anydbm.open
	96	except AttributeError:
	97	sys.stderr.write("anydbm module unavailable.\n")
	98	return 1
	99	elif opt in ("-g", "--gdbm"):
	100	try:
	101	dbopen = gdbm.open
	102	except AttributeError:
	103	sys.stderr.write("gdbm module unavailable.\n")
	104	return 1
	105	elif opt in ("-d", "--dbm"):
	106	try:
	107	dbopen = dbm.open
	108	except AttributeError:
	109	sys.stderr.write("dbm module unavailable.\n")
	110	return 1
	111	if dbopen is None:
	112	if bsddb is None:
	113	sys.stderr.write("bsddb module unavailable - ")
	114	sys.stderr.write("must specify dbtype.\n")
	115	return 1
	116	else:
	117	dbopen = bsddb.hashopen
	118
	119	try:
	120	db = dbopen(dbfile, 'r')
	121	except bsddb.error:
	122	sys.stderr.write("Unable to open %s. " % dbfile)
	123	sys.stderr.write("Check for format or version mismatch.\n")
	124	return 1
	125
	126	for k in db.keys():
	127	pickle.dump((k, db[k]), pfile, 1==1)
	128
	129	db.close()
	130	pfile.close()
	131
	132	return 0
	133
	134	if __name__ == "__main__":
	135	sys.exit(main(sys.argv[1:]))

sys/src/cmd/python/Tools/scripts/diff.py+49−0

@@ -0,0 +1,49 @@
	1	""" Command line interface to difflib.py providing diffs in four formats:
	2
	3	* ndiff: lists every line and highlights interline changes.
	4	* context: highlights clusters of changes in a before/after format.
	5	* unified: highlights clusters of changes in an inline format.
	6	* html: generates side by side comparison with change highlights.
	7
	8	"""
	9
	10	import sys, os, time, difflib, optparse
	11
	12	def main():
	13
	14	usage = "usage: %prog [options] fromfile tofile"
	15	parser = optparse.OptionParser(usage)
	16	parser.add_option("-c", action="store_true", default=False, help='Produce a context format diff (default)')
	17	parser.add_option("-u", action="store_true", default=False, help='Produce a unified format diff')
	18	parser.add_option("-m", action="store_true", default=False, help='Produce HTML side by side diff (can use -c and -l in conjunction)')
	19	parser.add_option("-n", action="store_true", default=False, help='Produce a ndiff format diff')
	20	parser.add_option("-l", "--lines", type="int", default=3, help='Set number of context lines (default 3)')
	21	(options, args) = parser.parse_args()
	22
	23	if len(args) == 0:
	24	parser.print_help()
	25	sys.exit(1)
	26	if len(args) != 2:
	27	parser.error("need to specify both a fromfile and tofile")
	28
	29	n = options.lines
	30	fromfile, tofile = args
	31
	32	fromdate = time.ctime(os.stat(fromfile).st_mtime)
	33	todate = time.ctime(os.stat(tofile).st_mtime)
	34	fromlines = open(fromfile, 'U').readlines()
	35	tolines = open(tofile, 'U').readlines()
	36
	37	if options.u:
	38	diff = difflib.unified_diff(fromlines, tolines, fromfile, tofile, fromdate, todate, n=n)
	39	elif options.n:
	40	diff = difflib.ndiff(fromlines, tolines)
	41	elif options.m:
	42	diff = difflib.HtmlDiff().make_file(fromlines,tolines,fromfile,tofile,context=options.c,numlines=n)
	43	else:
	44	diff = difflib.context_diff(fromlines, tolines, fromfile, tofile, fromdate, todate, n=n)
	45
	46	sys.stdout.writelines(diff)
	47
	48	if __name__ == '__main__':
	49	main()

sys/src/cmd/python/Tools/scripts/dutree.doc+54−0

@@ -0,0 +1,54 @@
	1	Path: cwi.nl!sun4nl!mcsun!uunet!cs.utexas.edu!convex!usenet
	2	From: tchrist@convex.COM (Tom Christiansen)
	3	Newsgroups: comp.lang.perl
	4	Subject: Re: The problems of Perl (Re: Question (silly?))
	5	Message-ID: <1992Jan17.053115.4220@convex.com>
	6	Date: 17 Jan 92 05:31:15 GMT
	7	References: <17458@ector.cs.purdue.edu> <1992Jan16.165347.25583@cherokee.uswest.com> <=#Hues+4@cs.psu.edu>
	8	Sender: usenet@convex.com (news access account)
	9	Reply-To: tchrist@convex.COM (Tom Christiansen)
	10	Organization: CONVEX Realtime Development, Colorado Springs, CO
	11	Lines: 83
	12	Nntp-Posting-Host: pixel.convex.com
	13
	14	From the keyboard of flee@cs.psu.edu (Felix Lee):
	15	:And Perl is definitely awkward with data types. I haven't yet found a
	16	:pleasant way of shoving non-trivial data types into Perl's grammar.
	17
	18	Yes, it's pretty aweful at that, alright. Sometimes I write perl programs
	19	that need them, and sometimes it just takes a little creativity. But
	20	sometimes it's not worth it. I actually wrote a C program the other day
	21	(gasp) because I didn't want to deal with a game matrix with six links per node.
	22
	23	:Here's a very simple problem that's tricky to express in Perl: process
	24	:the output of "du" to produce output that's indented to reflect the
	25	:tree structure, and with each subtree sorted by size. Something like:
	26	: 434 /etc
	27	: \| 344 .
	28	: \| 50 install
	29	: \| 35 uucp
	30	: \| 3 nserve
	31	: \| \| 2 .
	32	: \| \| 1 auth.info
	33	: \| 1 sm
	34	: \| 1 sm.bak
	35
	36	At first I thought I could just keep one local list around
	37	at once, but this seems inherently recursive. Which means
	38	I need an real recursive data structure. Maybe you could
	39	do it with one of the %assoc arrays Larry uses in the begat
	40	programs, but I broke down and got dirty. I think the hardest
	41	part was matching Felix's desired output exactly. It's not
	42	blazingly fast: I should probably inline the &childof routine,
	43	but it was faster to write than I could have written the
	44	equivalent C program.
	45
	46
	47	--tom
	48
	49	--
	50	"GUIs normally make it simple to accomplish simple actions and impossible
	51	to accomplish complex actions." --Doug Gwyn (22/Jun/91 in comp.unix.wizards)
	52
	53	Tom Christiansen tchrist@convex.com convex!tchrist
	54

sys/src/cmd/python/Tools/scripts/dutree.py+60−0

@@ -0,0 +1,60 @@
	1	#! /usr/bin/env python
	2	# Format du output in a tree shape
	3
	4	import os, sys, errno
	5
	6	def main():
	7	p = os.popen('du ' + ' '.join(sys.argv[1:]), 'r')
	8	total, d = None, {}
	9	for line in p.readlines():
	10	i = 0
	11	while line[i] in '0123456789': i = i+1
	12	size = eval(line[:i])
	13	while line[i] in ' \t': i = i+1
	14	filename = line[i:-1]
	15	comps = filename.split('/')
	16	if comps[0] == '': comps[0] = '/'
	17	if comps[len(comps)-1] == '': del comps[len(comps)-1]
	18	total, d = store(size, comps, total, d)
	19	try:
	20	display(total, d)
	21	except IOError, e:
	22	if e.errno != errno.EPIPE:
	23	raise
	24
	25	def store(size, comps, total, d):
	26	if comps == []:
	27	return size, d
	28	if not d.has_key(comps[0]):
	29	d[comps[0]] = None, {}
	30	t1, d1 = d[comps[0]]
	31	d[comps[0]] = store(size, comps[1:], t1, d1)
	32	return total, d
	33
	34	def display(total, d):
	35	show(total, d, '')
	36
	37	def show(total, d, prefix):
	38	if not d: return
	39	list = []
	40	sum = 0
	41	for key in d.keys():
	42	tsub, dsub = d[key]
	43	list.append((tsub, key))
	44	if tsub is not None: sum = sum + tsub
	45	## if sum < total:
	46	## list.append((total - sum, os.curdir))
	47	list.sort()
	48	list.reverse()
	49	width = len(repr(list[0][0]))
	50	for tsub, key in list:
	51	if tsub is None:
	52	psub = prefix
	53	else:
	54	print prefix + repr(tsub).rjust(width) + ' ' + key
	55	psub = prefix + ' '(width-1) + '\|' + ' '(len(key)+1)
	56	if d.has_key(key):
	57	show(tsub, d[key][1], psub)
	58
	59	if __name__ == '__main__':
	60	main()

sys/src/cmd/python/Tools/scripts/eptags.py+56−0

@@ -0,0 +1,56 @@
	1	#! /usr/bin/env python
	2	"""Create a TAGS file for Python programs, usable with GNU Emacs.
	3
	4	usage: eptags pyfiles...
	5
	6	The output TAGS file is usable with Emacs version 18, 19, 20.
	7	Tagged are:
	8	- functions (even inside other defs or classes)
	9	- classes
	10
	11	eptags warns about files it cannot open.
	12	eptags will not give warnings about duplicate tags.
	13
	14	BUGS:
	15	Because of tag duplication (methods with the same name in different
	16	classes), TAGS files are not very useful for most object-oriented
	17	python projects.
	18	"""
	19	import sys,re
	20
	21	expr = r'^[ \t](def\|class)[ \t]+([a-zA-Z_][a-zA-Z0-9_])[ \t]*[:\(]'
	22	matcher = re.compile(expr)
	23
	24	def treat_file(filename, outfp):
	25	"""Append tags found in file named 'filename' to the open file 'outfp'"""
	26	try:
	27	fp = open(filename, 'r')
	28	except:
	29	sys.stderr.write('Cannot open %s\n'%filename)
	30	return
	31	charno = 0
	32	lineno = 0
	33	tags = []
	34	size = 0
	35	while 1:
	36	line = fp.readline()
	37	if not line:
	38	break
	39	lineno = lineno + 1
	40	m = matcher.search(line)
	41	if m:
	42	tag = m.group(0) + '\177%d,%d\n' % (lineno, charno)
	43	tags.append(tag)
	44	size = size + len(tag)
	45	charno = charno + len(line)
	46	outfp.write('\f\n%s,%d\n' % (filename,size))
	47	for tag in tags:
	48	outfp.write(tag)
	49
	50	def main():
	51	outfp = open('TAGS', 'w')
	52	for filename in sys.argv[1:]:
	53	treat_file(filename, outfp)
	54
	55	if __name__=="__main__":
	56	main()

sys/src/cmd/python/Tools/scripts/finddiv.py+89−0

@@ -0,0 +1,89 @@
	1	#! /usr/bin/env python
	2
	3	"""finddiv - a grep-like tool that looks for division operators.
	4
	5	Usage: finddiv [-l] file_or_directory ...
	6
	7	For directory arguments, all files in the directory whose name ends in
	8	.py are processed, and subdirectories are processed recursively.
	9
	10	This actually tokenizes the files to avoid false hits in comments or
	11	strings literals.
	12
	13	By default, this prints all lines containing a / or /= operator, in
	14	grep -n style. With the -l option specified, it prints the filename
	15	of files that contain at least one / or /= operator.
	16	"""
	17
	18	import os
	19	import sys
	20	import getopt
	21	import tokenize
	22
	23	def main():
	24	try:
	25	opts, args = getopt.getopt(sys.argv[1:], "lh")
	26	except getopt.error, msg:
	27	usage(msg)
	28	return 2
	29	if not args:
	30	usage("at least one file argument is required")
	31	return 2
	32	listnames = 0
	33	for o, a in opts:
	34	if o == "-h":
	35	print __doc__
	36	return
	37	if o == "-l":
	38	listnames = 1
	39	exit = None
	40	for filename in args:
	41	x = process(filename, listnames)
	42	exit = exit or x
	43	return exit
	44
	45	def usage(msg):
	46	sys.stderr.write("%s: %s\n" % (sys.argv[0], msg))
	47	sys.stderr.write("Usage: %s [-l] file ...\n" % sys.argv[0])
	48	sys.stderr.write("Try `%s -h' for more information.\n" % sys.argv[0])
	49
	50	def process(filename, listnames):
	51	if os.path.isdir(filename):
	52	return processdir(filename, listnames)
	53	try:
	54	fp = open(filename)
	55	except IOError, msg:
	56	sys.stderr.write("Can't open: %s\n" % msg)
	57	return 1
	58	g = tokenize.generate_tokens(fp.readline)
	59	lastrow = None
	60	for type, token, (row, col), end, line in g:
	61	if token in ("/", "/="):
	62	if listnames:
	63	print filename
	64	break
	65	if row != lastrow:
	66	lastrow = row
	67	print "%s:%d:%s" % (filename, row, line),
	68	fp.close()
	69
	70	def processdir(dir, listnames):
	71	try:
	72	names = os.listdir(dir)
	73	except os.error, msg:
	74	sys.stderr.write("Can't list directory: %s\n" % dir)
	75	return 1
	76	files = []
	77	for name in names:
	78	fn = os.path.join(dir, name)
	79	if os.path.normcase(fn).endswith(".py") or os.path.isdir(fn):
	80	files.append(fn)
	81	files.sort(lambda a, b: cmp(os.path.normcase(a), os.path.normcase(b)))
	82	exit = None
	83	for fn in files:
	84	x = process(fn, listnames)
	85	exit = exit or x
	86	return exit
	87
	88	if __name__ == "__main__":
	89	sys.exit(main())

sys/src/cmd/python/Tools/scripts/findlinksto.py+43−0

@@ -0,0 +1,43 @@
	1	#! /usr/bin/env python
	2
	3	# findlinksto
	4	#
	5	# find symbolic links to a path matching a regular expression
	6
	7	import os
	8	import sys
	9	import re
	10	import getopt
	11
	12	def main():
	13	try:
	14	opts, args = getopt.getopt(sys.argv[1:], '')
	15	if len(args) < 2:
	16	raise getopt.GetoptError('not enough arguments', None)
	17	except getopt.GetoptError, msg:
	18	sys.stdout = sys.stderr
	19	print msg
	20	print 'usage: findlinksto pattern directory ...'
	21	sys.exit(2)
	22	pat, dirs = args[0], args[1:]
	23	prog = re.compile(pat)
	24	for dirname in dirs:
	25	os.path.walk(dirname, visit, prog)
	26
	27	def visit(prog, dirname, names):
	28	if os.path.islink(dirname):
	29	names[:] = []
	30	return
	31	if os.path.ismount(dirname):
	32	print 'descend into', dirname
	33	for name in names:
	34	name = os.path.join(dirname, name)
	35	try:
	36	linkto = os.readlink(name)
	37	if prog.search(linkto) is not None:
	38	print name, '->', linkto
	39	except os.error:
	40	pass
	41
	42	if __name__ == '__main__':
	43	main()

sys/src/cmd/python/Tools/scripts/findnocoding.py+104−0

@@ -0,0 +1,104 @@
	1	#!/usr/bin/env python
	2
	3	"""List all those Python files that require a coding directive
	4
	5	Usage: nocoding.py dir1 [dir2...]
	6	"""
	7
	8	__author__ = "Oleg Broytmann, Georg Brandl"
	9
	10	import sys, os, re, getopt
	11
	12	# our pysource module finds Python source files
	13	try:
	14	import pysource
	15	except:
	16	# emulate the module with a simple os.walk
	17	class pysource:
	18	has_python_ext = looks_like_python = can_be_compiled = None
	19	def walk_python_files(self, paths, args, *kwargs):
	20	for path in paths:
	21	if os.path.isfile(path):
	22	yield path.endswith(".py")
	23	elif os.path.isdir(path):
	24	for root, dirs, files in os.walk(path):
	25	for filename in files:
	26	if filename.endswith(".py"):
	27	yield os.path.join(root, filename)
	28	pysource = pysource()
	29
	30
	31	print >>sys.stderr, ("The pysource module is not available; "
	32	"no sophisticated Python source file search will be done.")
	33
	34
	35	decl_re = re.compile(r"coding[=:]\s*([-\w.]+)")
	36
	37	def get_declaration(line):
	38	match = decl_re.search(line)
	39	if match:
	40	return match.group(1)
	41	return ''
	42
	43	def has_correct_encoding(text, codec):
	44	try:
	45	unicode(text, codec)
	46	except UnicodeDecodeError:
	47	return False
	48	else:
	49	return True
	50
	51	def needs_declaration(fullpath):
	52	try:
	53	infile = open(fullpath, 'rU')
	54	except IOError: # Oops, the file was removed - ignore it
	55	return None
	56
	57	line1 = infile.readline()
	58	line2 = infile.readline()
	59
	60	if get_declaration(line1) or get_declaration(line2):
	61	# the file does have an encoding declaration, so trust it
	62	infile.close()
	63	return False
	64
	65	# check the whole file for non-ASCII characters
	66	rest = infile.read()
	67	infile.close()
	68
	69	if has_correct_encoding(line1+line2+rest, "ascii"):
	70	return False
	71
	72	return True
	73
	74
	75	usage = """Usage: %s [-cd] paths...
	76	-c: recognize Python source files trying to compile them
	77	-d: debug output""" % sys.argv[0]
	78
	79	try:
	80	opts, args = getopt.getopt(sys.argv[1:], 'cd')
	81	except getopt.error, msg:
	82	print >>sys.stderr, msg
	83	print >>sys.stderr, usage
	84	sys.exit(1)
	85
	86	is_python = pysource.looks_like_python
	87	debug = False
	88
	89	for o, a in opts:
	90	if o == '-c':
	91	is_python = pysource.can_be_compiled
	92	elif o == '-d':
	93	debug = True
	94
	95	if not args:
	96	print >>sys.stderr, usage
	97	sys.exit(1)
	98
	99	for fullpath in pysource.walk_python_files(args, is_python):
	100	if debug:
	101	print "Testing for coding: %s" % fullpath
	102	result = needs_declaration(fullpath)
	103	if result:
	104	print fullpath

sys/src/cmd/python/Tools/scripts/fixcid.py+314−0

@@ -0,0 +1,314 @@
	1	#! /usr/bin/env python
	2
	3	# Perform massive identifier substitution on C source files.
	4	# This actually tokenizes the files (to some extent) so it can
	5	# avoid making substitutions inside strings or comments.
	6	# Inside strings, substitutions are never made; inside comments,
	7	# it is a user option (off by default).
	8	#
	9	# The substitutions are read from one or more files whose lines,
	10	# when not empty, after stripping comments starting with #,
	11	# must contain exactly two words separated by whitespace: the
	12	# old identifier and its replacement.
	13	#
	14	# The option -r reverses the sense of the substitutions (this may be
	15	# useful to undo a particular substitution).
	16	#
	17	# If the old identifier is prefixed with a '*' (with no intervening
	18	# whitespace), then it will not be substituted inside comments.
	19	#
	20	# Command line arguments are files or directories to be processed.
	21	# Directories are searched recursively for files whose name looks
	22	# like a C file (ends in .h or .c). The special filename '-' means
	23	# operate in filter mode: read stdin, write stdout.
	24	#
	25	# Symbolic links are always ignored (except as explicit directory
	26	# arguments).
	27	#
	28	# The original files are kept as back-up with a "~" suffix.
	29	#
	30	# Changes made are reported to stdout in a diff-like format.
	31	#
	32	# NB: by changing only the function fixline() you can turn this
	33	# into a program for different changes to C source files; by
	34	# changing the function wanted() you can make a different selection of
	35	# files.
	36
	37	import sys
	38	import re
	39	import os
	40	from stat import *
	41	import getopt
	42
	43	err = sys.stderr.write
	44	dbg = err
	45	rep = sys.stdout.write
	46
	47	def usage():
	48	progname = sys.argv[0]
	49	err('Usage: ' + progname +
	50	' [-c] [-r] [-s file] ... file-or-directory ...\n')
	51	err('\n')
	52	err('-c : substitute inside comments\n')
	53	err('-r : reverse direction for following -s options\n')
	54	err('-s substfile : add a file of substitutions\n')
	55	err('\n')
	56	err('Each non-empty non-comment line in a substitution file must\n')
	57	err('contain exactly two words: an identifier and its replacement.\n')
	58	err('Comments start with a # character and end at end of line.\n')
	59	err('If an identifier is preceded with a *, it is not substituted\n')
	60	err('inside a comment even when -c is specified.\n')
	61
	62	def main():
	63	try:
	64	opts, args = getopt.getopt(sys.argv[1:], 'crs:')
	65	except getopt.error, msg:
	66	err('Options error: ' + str(msg) + '\n')
	67	usage()
	68	sys.exit(2)
	69	bad = 0
	70	if not args: # No arguments
	71	usage()
	72	sys.exit(2)
	73	for opt, arg in opts:
	74	if opt == '-c':
	75	setdocomments()
	76	if opt == '-r':
	77	setreverse()
	78	if opt == '-s':
	79	addsubst(arg)
	80	for arg in args:
	81	if os.path.isdir(arg):
	82	if recursedown(arg): bad = 1
	83	elif os.path.islink(arg):
	84	err(arg + ': will not process symbolic links\n')
	85	bad = 1
	86	else:
	87	if fix(arg): bad = 1
	88	sys.exit(bad)
	89
	90	# Change this regular expression to select a different set of files
	91	Wanted = '^[a-zA-Z0-9_]+\.[ch]$'
	92	def wanted(name):
	93	return re.match(Wanted, name) >= 0
	94
	95	def recursedown(dirname):
	96	dbg('recursedown(%r)\n' % (dirname,))
	97	bad = 0
	98	try:
	99	names = os.listdir(dirname)
	100	except os.error, msg:
	101	err(dirname + ': cannot list directory: ' + str(msg) + '\n')
	102	return 1
	103	names.sort()
	104	subdirs = []
	105	for name in names:
	106	if name in (os.curdir, os.pardir): continue
	107	fullname = os.path.join(dirname, name)
	108	if os.path.islink(fullname): pass
	109	elif os.path.isdir(fullname):
	110	subdirs.append(fullname)
	111	elif wanted(name):
	112	if fix(fullname): bad = 1
	113	for fullname in subdirs:
	114	if recursedown(fullname): bad = 1
	115	return bad
	116
	117	def fix(filename):
	118	## dbg('fix(%r)\n' % (filename,))
	119	if filename == '-':
	120	# Filter mode
	121	f = sys.stdin
	122	g = sys.stdout
	123	else:
	124	# File replacement mode
	125	try:
	126	f = open(filename, 'r')
	127	except IOError, msg:
	128	err(filename + ': cannot open: ' + str(msg) + '\n')
	129	return 1
	130	head, tail = os.path.split(filename)
	131	tempname = os.path.join(head, '@' + tail)
	132	g = None
	133	# If we find a match, we rewind the file and start over but
	134	# now copy everything to a temp file.
	135	lineno = 0
	136	initfixline()
	137	while 1:
	138	line = f.readline()
	139	if not line: break
	140	lineno = lineno + 1
	141	while line[-2:] == '\\\n':
	142	nextline = f.readline()
	143	if not nextline: break
	144	line = line + nextline
	145	lineno = lineno + 1
	146	newline = fixline(line)
	147	if newline != line:
	148	if g is None:
	149	try:
	150	g = open(tempname, 'w')
	151	except IOError, msg:
	152	f.close()
	153	err(tempname+': cannot create: '+
	154	str(msg)+'\n')
	155	return 1
	156	f.seek(0)
	157	lineno = 0
	158	initfixline()
	159	rep(filename + ':\n')
	160	continue # restart from the beginning
	161	rep(repr(lineno) + '\n')
	162	rep('< ' + line)
	163	rep('> ' + newline)
	164	if g is not None:
	165	g.write(newline)
	166
	167	# End of file
	168	if filename == '-': return 0 # Done in filter mode
	169	f.close()
	170	if not g: return 0 # No changes
	171
	172	# Finishing touch -- move files
	173
	174	# First copy the file's mode to the temp file
	175	try:
	176	statbuf = os.stat(filename)
	177	os.chmod(tempname, statbuf[ST_MODE] & 07777)
	178	except os.error, msg:
	179	err(tempname + ': warning: chmod failed (' + str(msg) + ')\n')
	180	# Then make a backup of the original file as filename~
	181	try:
	182	os.rename(filename, filename + '~')
	183	except os.error, msg:
	184	err(filename + ': warning: backup failed (' + str(msg) + ')\n')
	185	# Now move the temp file to the original file
	186	try:
	187	os.rename(tempname, filename)
	188	except os.error, msg:
	189	err(filename + ': rename failed (' + str(msg) + ')\n')
	190	return 1
	191	# Return succes
	192	return 0
	193
	194	# Tokenizing ANSI C (partly)
	195
	196	Identifier = '$struct $?[a-zA-Z_][a-zA-Z0-9_]+'
	197	String = '"$[^\n\\"]\\|\\\\.$*"'
	198	Char = '\'$[^\n\\\']\\|\\\\.$*\''
	199	CommentStart = '/\*'
	200	CommentEnd = '\*/'
	201
	202	Hexnumber = '0[xX][0-9a-fA-F][uUlL]'
	203	Octnumber = '0[0-7][uUlL]'
	204	Decnumber = '[1-9][0-9][uUlL]'
	205	Intnumber = Hexnumber + '\\|' + Octnumber + '\\|' + Decnumber
	206	Exponent = '[eE][-+]?[0-9]+'
	207	Pointfloat = '$[0-9]+\.[0-9]*\\|\.[0-9]+$$' + Exponent + '$?'
	208	Expfloat = '[0-9]+' + Exponent
	209	Floatnumber = Pointfloat + '\\|' + Expfloat
	210	Number = Floatnumber + '\\|' + Intnumber
	211
	212	# Anything else is an operator -- don't list this explicitly because of '/*'
	213
	214	OutsideComment = (Identifier, Number, String, Char, CommentStart)
	215	OutsideCommentPattern = '(' + '\|'.join(OutsideComment) + ')'
	216	OutsideCommentProgram = re.compile(OutsideCommentPattern)
	217
	218	InsideComment = (Identifier, Number, CommentEnd)
	219	InsideCommentPattern = '(' + '\|'.join(InsideComment) + ')'
	220	InsideCommentProgram = re.compile(InsideCommentPattern)
	221
	222	def initfixline():
	223	global Program
	224	Program = OutsideCommentProgram
	225
	226	def fixline(line):
	227	global Program
	228	## print '-->', repr(line)
	229	i = 0
	230	while i < len(line):
	231	i = Program.search(line, i)
	232	if i < 0: break
	233	found = Program.group(0)
	234	## if Program is InsideCommentProgram: print '...',
	235	## else: print ' ',
	236	## print found
	237	if len(found) == 2:
	238	if found == '/*':
	239	Program = InsideCommentProgram
	240	elif found == '*/':
	241	Program = OutsideCommentProgram
	242	n = len(found)
	243	if Dict.has_key(found):
	244	subst = Dict[found]
	245	if Program is InsideCommentProgram:
	246	if not Docomments:
	247	print 'Found in comment:', found
	248	i = i + n
	249	continue
	250	if NotInComment.has_key(found):
	251	## print 'Ignored in comment:',
	252	## print found, '-->', subst
	253	## print 'Line:', line,
	254	subst = found
	255	## else:
	256	## print 'Substituting in comment:',
	257	## print found, '-->', subst
	258	## print 'Line:', line,
	259	line = line[:i] + subst + line[i+n:]
	260	n = len(subst)
	261	i = i + n
	262	return line
	263
	264	Docomments = 0
	265	def setdocomments():
	266	global Docomments
	267	Docomments = 1
	268
	269	Reverse = 0
	270	def setreverse():
	271	global Reverse
	272	Reverse = (not Reverse)
	273
	274	Dict = {}
	275	NotInComment = {}
	276	def addsubst(substfile):
	277	try:
	278	fp = open(substfile, 'r')
	279	except IOError, msg:
	280	err(substfile + ': cannot read substfile: ' + str(msg) + '\n')
	281	sys.exit(1)
	282	lineno = 0
	283	while 1:
	284	line = fp.readline()
	285	if not line: break
	286	lineno = lineno + 1
	287	try:
	288	i = line.index('#')
	289	except ValueError:
	290	i = -1 # Happens to delete trailing \n
	291	words = line[:i].split()
	292	if not words: continue
	293	if len(words) == 3 and words[0] == 'struct':
	294	words[:2] = [words[0] + ' ' + words[1]]
	295	elif len(words) <> 2:
	296	err(substfile + '%s:%r: warning: bad line: %r' % (substfile, lineno, line))
	297	continue
	298	if Reverse:
	299	[value, key] = words
	300	else:
	301	[key, value] = words
	302	if value[0] == '*':
	303	value = value[1:]
	304	if key[0] == '*':
	305	key = key[1:]
	306	NotInComment[key] = value
	307	if Dict.has_key(key):
	308	err('%s:%r: warning: overriding: %r %r\n' % (substfile, lineno, key, value))
	309	err('%s:%r: warning: previous: %r\n' % (substfile, lineno, Dict[key]))
	310	Dict[key] = value
	311	fp.close()
	312
	313	if __name__ == '__main__':
	314	main()

sys/src/cmd/python/Tools/scripts/fixdiv.py+380−0

@@ -0,0 +1,380 @@
	1	#! /usr/bin/env python
	2
	3	"""fixdiv - tool to fix division operators.
	4
	5	To use this tool, first run `python -Qwarnall yourscript.py 2>warnings'.
	6	This runs the script `yourscript.py' while writing warning messages
	7	about all uses of the classic division operator to the file
	8	`warnings'. The warnings look like this:
	9
	10	<file>:<line>: DeprecationWarning: classic <type> division
	11
	12	The warnings are written to stderr, so you must use `2>' for the I/O
	13	redirect. I know of no way to redirect stderr on Windows in a DOS
	14	box, so you will have to modify the script to set sys.stderr to some
	15	kind of log file if you want to do this on Windows.
	16
	17	The warnings are not limited to the script; modules imported by the
	18	script may also trigger warnings. In fact a useful technique is to
	19	write a test script specifically intended to exercise all code in a
	20	particular module or set of modules.
	21
	22	Then run `python fixdiv.py warnings'. This first reads the warnings,
	23	looking for classic division warnings, and sorts them by file name and
	24	line number. Then, for each file that received at least one warning,
	25	it parses the file and tries to match the warnings up to the division
	26	operators found in the source code. If it is successful, it writes
	27	its findings to stdout, preceded by a line of dashes and a line of the
	28	form:
	29
	30	Index: <file>
	31
	32	If the only findings found are suggestions to change a / operator into
	33	a // operator, the output is acceptable input for the Unix 'patch'
	34	program.
	35
	36	Here are the possible messages on stdout (N stands for a line number):
	37
	38	- A plain-diff-style change ('NcN', a line marked by '<', a line
	39	containing '---', and a line marked by '>'):
	40
	41	A / operator was found that should be changed to //. This is the
	42	recommendation when only int and/or long arguments were seen.
	43
	44	- 'True division / operator at line N' and a line marked by '=':
	45
	46	A / operator was found that can remain unchanged. This is the
	47	recommendation when only float and/or complex arguments were seen.
	48
	49	- 'Ambiguous / operator (..., ...) at line N', line marked by '?':
	50
	51	A / operator was found for which int or long as well as float or
	52	complex arguments were seen. This is highly unlikely; if it occurs,
	53	you may have to restructure the code to keep the classic semantics,
	54	or maybe you don't care about the classic semantics.
	55
	56	- 'No conclusive evidence on line N', line marked by '*':
	57
	58	A / operator was found for which no warnings were seen. This could
	59	be code that was never executed, or code that was only executed
	60	with user-defined objects as arguments. You will have to
	61	investigate further. Note that // can be overloaded separately from
	62	/, using __floordiv__. True division can also be separately
	63	overloaded, using __truediv__. Classic division should be the same
	64	as either of those. (XXX should I add a warning for division on
	65	user-defined objects, to disambiguate this case from code that was
	66	never executed?)
	67
	68	- 'Phantom ... warnings for line N', line marked by '*':
	69
	70	A warning was seen for a line not containing a / operator. The most
	71	likely cause is a warning about code executed by 'exec' or eval()
	72	(see note below), or an indirect invocation of the / operator, for
	73	example via the div() function in the operator module. It could
	74	also be caused by a change to the file between the time the test
	75	script was run to collect warnings and the time fixdiv was run.
	76
	77	- 'More than one / operator in line N'; or
	78	'More than one / operator per statement in lines N-N':
	79
	80	The scanner found more than one / operator on a single line, or in a
	81	statement split across multiple lines. Because the warnings
	82	framework doesn't (and can't) show the offset within the line, and
	83	the code generator doesn't always give the correct line number for
	84	operations in a multi-line statement, we can't be sure whether all
	85	operators in the statement were executed. To be on the safe side,
	86	by default a warning is issued about this case. In practice, these
	87	cases are usually safe, and the -m option suppresses these warning.
	88
	89	- 'Can't find the / operator in line N', line marked by '*':
	90
	91	This really shouldn't happen. It means that the tokenize module
	92	reported a '/' operator but the line it returns didn't contain a '/'
	93	character at the indicated position.
	94
	95	- 'Bad warning for line N: XYZ', line marked by '*':
	96
	97	This really shouldn't happen. It means that a 'classic XYZ
	98	division' warning was read with XYZ being something other than
	99	'int', 'long', 'float', or 'complex'.
	100
	101	Notes:
	102
	103	- The augmented assignment operator /= is handled the same way as the
	104	/ operator.
	105
	106	- This tool never looks at the // operator; no warnings are ever
	107	generated for use of this operator.
	108
	109	- This tool never looks at the / operator when a future division
	110	statement is in effect; no warnings are generated in this case, and
	111	because the tool only looks at files for which at least one classic
	112	division warning was seen, it will never look at files containing a
	113	future division statement.
	114
	115	- Warnings may be issued for code not read from a file, but executed
	116	using an exec statement or the eval() function. These may have
	117	<string> in the filename position, in which case the fixdiv script
	118	will attempt and fail to open a file named '<string>' and issue a
	119	warning about this failure; or these may be reported as 'Phantom'
	120	warnings (see above). You're on your own to deal with these. You
	121	could make all recommended changes and add a future division
	122	statement to all affected files, and then re-run the test script; it
	123	should not issue any warnings. If there are any, and you have a
	124	hard time tracking down where they are generated, you can use the
	125	-Werror option to force an error instead of a first warning,
	126	generating a traceback.
	127
	128	- The tool should be run from the same directory as that from which
	129	the original script was run, otherwise it won't be able to open
	130	files given by relative pathnames.
	131	"""
	132
	133	import sys
	134	import getopt
	135	import re
	136	import tokenize
	137
	138	multi_ok = 0
	139
	140	def main():
	141	try:
	142	opts, args = getopt.getopt(sys.argv[1:], "hm")
	143	except getopt.error, msg:
	144	usage(msg)
	145	return 2
	146	for o, a in opts:
	147	if o == "-h":
	148	print __doc__
	149	return
	150	if o == "-m":
	151	global multi_ok
	152	multi_ok = 1
	153	if not args:
	154	usage("at least one file argument is required")
	155	return 2
	156	if args[1:]:
	157	sys.stderr.write("%s: extra file arguments ignored\n", sys.argv[0])
	158	warnings = readwarnings(args[0])
	159	if warnings is None:
	160	return 1
	161	files = warnings.keys()
	162	if not files:
	163	print "No classic division warnings read from", args[0]
	164	return
	165	files.sort()
	166	exit = None
	167	for filename in files:
	168	x = process(filename, warnings[filename])
	169	exit = exit or x
	170	return exit
	171
	172	def usage(msg):
	173	sys.stderr.write("%s: %s\n" % (sys.argv[0], msg))
	174	sys.stderr.write("Usage: %s [-m] warnings\n" % sys.argv[0])
	175	sys.stderr.write("Try `%s -h' for more information.\n" % sys.argv[0])
	176
	177	PATTERN = ("^(.+?):(\d+): DeprecationWarning: "
	178	"classic (int\|long\|float\|complex) division$")
	179
	180	def readwarnings(warningsfile):
	181	prog = re.compile(PATTERN)
	182	try:
	183	f = open(warningsfile)
	184	except IOError, msg:
	185	sys.stderr.write("can't open: %s\n" % msg)
	186	return
	187	warnings = {}
	188	while 1:
	189	line = f.readline()
	190	if not line:
	191	break
	192	m = prog.match(line)
	193	if not m:
	194	if line.find("division") >= 0:
	195	sys.stderr.write("Warning: ignored input " + line)
	196	continue
	197	filename, lineno, what = m.groups()
	198	list = warnings.get(filename)
	199	if list is None:
	200	warnings[filename] = list = []
	201	list.append((int(lineno), intern(what)))
	202	f.close()
	203	return warnings
	204
	205	def process(filename, list):
	206	print "-"*70
	207	assert list # if this fails, readwarnings() is broken
	208	try:
	209	fp = open(filename)
	210	except IOError, msg:
	211	sys.stderr.write("can't open: %s\n" % msg)
	212	return 1
	213	print "Index:", filename
	214	f = FileContext(fp)
	215	list.sort()
	216	index = 0 # list[:index] has been processed, list[index:] is still to do
	217	g = tokenize.generate_tokens(f.readline)
	218	while 1:
	219	startlineno, endlineno, slashes = lineinfo = scanline(g)
	220	if startlineno is None:
	221	break
	222	assert startlineno <= endlineno is not None
	223	orphans = []
	224	while index < len(list) and list[index][0] < startlineno:
	225	orphans.append(list[index])
	226	index += 1
	227	if orphans:
	228	reportphantomwarnings(orphans, f)
	229	warnings = []
	230	while index < len(list) and list[index][0] <= endlineno:
	231	warnings.append(list[index])
	232	index += 1
	233	if not slashes and not warnings:
	234	pass
	235	elif slashes and not warnings:
	236	report(slashes, "No conclusive evidence")
	237	elif warnings and not slashes:
	238	reportphantomwarnings(warnings, f)
	239	else:
	240	if len(slashes) > 1:
	241	if not multi_ok:
	242	rows = []
	243	lastrow = None
	244	for (row, col), line in slashes:
	245	if row == lastrow:
	246	continue
	247	rows.append(row)
	248	lastrow = row
	249	assert rows
	250	if len(rows) == 1:
	251	print "*** More than one / operator in line", rows[0]
	252	else:
	253	print "*** More than one / operator per statement",
	254	print "in lines %d-%d" % (rows[0], rows[-1])
	255	intlong = []
	256	floatcomplex = []
	257	bad = []
	258	for lineno, what in warnings:
	259	if what in ("int", "long"):
	260	intlong.append(what)
	261	elif what in ("float", "complex"):
	262	floatcomplex.append(what)
	263	else:
	264	bad.append(what)
	265	lastrow = None
	266	for (row, col), line in slashes:
	267	if row == lastrow:
	268	continue
	269	lastrow = row
	270	line = chop(line)
	271	if line[col:col+1] != "/":
	272	print "*** Can't find the / operator in line %d:" % row
	273	print "*", line
	274	continue
	275	if bad:
	276	print "*** Bad warning for line %d:" % row, bad
	277	print "*", line
	278	elif intlong and not floatcomplex:
	279	print "%dc%d" % (row, row)
	280	print "<", line
	281	print "---"
	282	print ">", line[:col] + "/" + line[col:]
	283	elif floatcomplex and not intlong:
	284	print "True division / operator at line %d:" % row
	285	print "=", line
	286	elif intlong and floatcomplex:
	287	print "*** Ambiguous / operator (%s, %s) at line %d:" % (
	288	"\|".join(intlong), "\|".join(floatcomplex), row)
	289	print "?", line
	290	fp.close()
	291
	292	def reportphantomwarnings(warnings, f):
	293	blocks = []
	294	lastrow = None
	295	lastblock = None
	296	for row, what in warnings:
	297	if row != lastrow:
	298	lastblock = [row]
	299	blocks.append(lastblock)
	300	lastblock.append(what)
	301	for block in blocks:
	302	row = block[0]
	303	whats = "/".join(block[1:])
	304	print "*** Phantom %s warnings for line %d:" % (whats, row)
	305	f.report(row, mark="*")
	306
	307	def report(slashes, message):
	308	lastrow = None
	309	for (row, col), line in slashes:
	310	if row != lastrow:
	311	print "*** %s on line %d:" % (message, row)
	312	print "*", chop(line)
	313	lastrow = row
	314
	315	class FileContext:
	316	def __init__(self, fp, window=5, lineno=1):
	317	self.fp = fp
	318	self.window = 5
	319	self.lineno = 1
	320	self.eoflookahead = 0
	321	self.lookahead = []
	322	self.buffer = []
	323	def fill(self):
	324	while len(self.lookahead) < self.window and not self.eoflookahead:
	325	line = self.fp.readline()
	326	if not line:
	327	self.eoflookahead = 1
	328	break
	329	self.lookahead.append(line)
	330	def readline(self):
	331	self.fill()
	332	if not self.lookahead:
	333	return ""
	334	line = self.lookahead.pop(0)
	335	self.buffer.append(line)
	336	self.lineno += 1
	337	return line
	338	def truncate(self):
	339	del self.buffer[-window:]
	340	def __getitem__(self, index):
	341	self.fill()
	342	bufstart = self.lineno - len(self.buffer)
	343	lookend = self.lineno + len(self.lookahead)
	344	if bufstart <= index < self.lineno:
	345	return self.buffer[index - bufstart]
	346	if self.lineno <= index < lookend:
	347	return self.lookahead[index - self.lineno]
	348	raise KeyError
	349	def report(self, first, last=None, mark="*"):
	350	if last is None:
	351	last = first
	352	for i in range(first, last+1):
	353	try:
	354	line = self[first]
	355	except KeyError:
	356	line = "<missing line>"
	357	print mark, chop(line)
	358
	359	def scanline(g):
	360	slashes = []
	361	startlineno = None
	362	endlineno = None
	363	for type, token, start, end, line in g:
	364	endlineno = end[0]
	365	if startlineno is None:
	366	startlineno = endlineno
	367	if token in ("/", "/="):
	368	slashes.append((start, line))
	369	if type == tokenize.NEWLINE:
	370	break
	371	return startlineno, endlineno, slashes
	372
	373	def chop(line):
	374	if line.endswith("\n"):
	375	return line[:-1]
	376	else:
	377	return line
	378
	379	if __name__ == "__main__":
	380	sys.exit(main())

sys/src/cmd/python/Tools/scripts/fixheader.py+49−0

@@ -0,0 +1,49 @@
	1	#! /usr/bin/env python
	2
	3	# Add some standard cpp magic to a header file
	4
	5	import sys
	6
	7	def main():
	8	args = sys.argv[1:]
	9	for filename in args:
	10	process(filename)
	11
	12	def process(filename):
	13	try:
	14	f = open(filename, 'r')
	15	except IOError, msg:
	16	sys.stderr.write('%s: can\'t open: %s\n' % (filename, str(msg)))
	17	return
	18	data = f.read()
	19	f.close()
	20	if data[:2] <> '/*':
	21	sys.stderr.write('%s does not begin with C comment\n' % filename)
	22	return
	23	try:
	24	f = open(filename, 'w')
	25	except IOError, msg:
	26	sys.stderr.write('%s: can\'t write: %s\n' % (filename, str(msg)))
	27	return
	28	sys.stderr.write('Processing %s ...\n' % filename)
	29	magic = 'Py_'
	30	for c in filename:
	31	if ord(c)<=0x80 and c.isalnum():
	32	magic = magic + c.upper()
	33	else: magic = magic + '_'
	34	sys.stdout = f
	35	print '#ifndef', magic
	36	print '#define', magic
	37	print '#ifdef __cplusplus'
	38	print 'extern "C" {'
	39	print '#endif'
	40	print
	41	f.write(data)
	42	print
	43	print '#ifdef __cplusplus'
	44	print '}'
	45	print '#endif'
	46	print '#endif /', '!'+magic, '/'
	47
	48	if __name__ == '__main__':
	49	main()

sys/src/cmd/python/Tools/scripts/fixnotice.py+113−0

@@ -0,0 +1,113 @@
	1	#! /usr/bin/env python
	2
	3	"""(Ostensibly) fix copyright notices in files.
	4
	5	Actually, this sript will simply replace a block of text in a file from one
	6	string to another. It will only do this once though, i.e. not globally
	7	throughout the file. It writes a backup file and then does an os.rename()
	8	dance for atomicity.
	9
	10	Usage: fixnotices.py [options] [filenames]
	11	Options:
	12	-h / --help
	13	Print this message and exit
	14
	15	--oldnotice=file
	16	Use the notice in the file as the old (to be replaced) string, instead
	17	of the hard coded value in the script.
	18
	19	--newnotice=file
	20	Use the notice in the file as the new (replacement) string, instead of
	21	the hard coded value in the script.
	22
	23	--dry-run
	24	Don't actually make the changes, but print out the list of files that
	25	would change. When used with -v, a status will be printed for every
	26	file.
	27
	28	-v / --verbose
	29	Print a message for every file looked at, indicating whether the file
	30	is changed or not.
	31	"""
	32
	33	OLD_NOTICE = """/***********************************************************
	34	Copyright (c) 2000, BeOpen.com.
	35	Copyright (c) 1995-2000, Corporation for National Research Initiatives.
	36	Copyright (c) 1990-1995, Stichting Mathematisch Centrum.
	37	All rights reserved.
	38
	39	See the file "Misc/COPYRIGHT" for information on usage and
	40	redistribution of this file, and for a DISCLAIMER OF ALL WARRANTIES.
	41	******************************************************************/
	42	"""
	43	import os
	44	import sys
	45	import getopt
	46
	47	NEW_NOTICE = ""
	48	DRYRUN = 0
	49	VERBOSE = 0
	50
	51
	52	def usage(code, msg=''):
	53	print __doc__ % globals()
	54	if msg:
	55	print msg
	56	sys.exit(code)
	57
	58
	59	def main():
	60	global DRYRUN, OLD_NOTICE, NEW_NOTICE, VERBOSE
	61	try:
	62	opts, args = getopt.getopt(sys.argv[1:], 'hv',
	63	['help', 'oldnotice=', 'newnotice=',
	64	'dry-run', 'verbose'])
	65	except getopt.error, msg:
	66	usage(1, msg)
	67
	68	for opt, arg in opts:
	69	if opt in ('-h', '--help'):
	70	usage(0)
	71	elif opt in ('-v', '--verbose'):
	72	VERBOSE = 1
	73	elif opt == '--dry-run':
	74	DRYRUN = 1
	75	elif opt == '--oldnotice':
	76	fp = open(arg)
	77	OLD_NOTICE = fp.read()
	78	fp.close()
	79	elif opt == '--newnotice':
	80	fp = open(arg)
	81	NEW_NOTICE = fp.read()
	82	fp.close()
	83
	84	for arg in args:
	85	process(arg)
	86
	87
	88	def process(file):
	89	f = open(file)
	90	data = f.read()
	91	f.close()
	92	i = data.find(OLD_NOTICE)
	93	if i < 0:
	94	if VERBOSE:
	95	print 'no change:', file
	96	return
	97	elif DRYRUN or VERBOSE:
	98	print ' change:', file
	99	if DRYRUN:
	100	# Don't actually change the file
	101	return
	102	data = data[:i] + NEW_NOTICE + data[i+len(OLD_NOTICE):]
	103	new = file + ".new"
	104	backup = file + ".bak"
	105	f = open(new, "w")
	106	f.write(data)
	107	f.close()
	108	os.rename(file, backup)
	109	os.rename(new, file)
	110
	111
	112	if __name__ == '__main__':
	113	main()

sys/src/cmd/python/Tools/scripts/fixps.py+33−0

@@ -0,0 +1,33 @@
	1	#!/usr/bin/env python
	2
	3	# Fix Python script(s) to reference the interpreter via /usr/bin/env python.
	4	# Warning: this overwrites the file without making a backup.
	5
	6	import sys
	7	import re
	8
	9
	10	def main():
	11	for filename in sys.argv[1:]:
	12	try:
	13	f = open(filename, 'r')
	14	except IOError, msg:
	15	print filename, ': can\'t open :', msg
	16	continue
	17	line = f.readline()
	18	if not re.match('^#! */usr/local/bin/python', line):
	19	print filename, ': not a /usr/local/bin/python script'
	20	f.close()
	21	continue
	22	rest = f.read()
	23	f.close()
	24	line = re.sub('/usr/local/bin/python',
	25	'/usr/bin/env python', line)
	26	print filename, ':', repr(line)
	27	f = open(filename, "w")
	28	f.write(line)
	29	f.write(rest)
	30	f.close()
	31
	32	if __name__ == '__main__':
	33	main()

sys/src/cmd/python/Tools/scripts/ftpmirror.py+400−0

@@ -0,0 +1,400 @@
	1	#! /usr/bin/env python
	2
	3	"""Mirror a remote ftp subtree into a local directory tree.
	4
	5	usage: ftpmirror [-v] [-q] [-i] [-m] [-n] [-r] [-s pat]
	6	[-l username [-p passwd [-a account]]]
	7	hostname[:port] [remotedir [localdir]]
	8	-v: verbose
	9	-q: quiet
	10	-i: interactive mode
	11	-m: macintosh server (NCSA telnet 2.4) (implies -n -s '*.o')
	12	-n: don't log in
	13	-r: remove local files/directories no longer pertinent
	14	-l username [-p passwd [-a account]]: login info (default .netrc or anonymous)
	15	-s pat: skip files matching pattern
	16	hostname: remote host w/ optional port separated by ':'
	17	remotedir: remote directory (default initial)
	18	localdir: local directory (default current)
	19	"""
	20
	21	import os
	22	import sys
	23	import time
	24	import getopt
	25	import ftplib
	26	import netrc
	27	from fnmatch import fnmatch
	28
	29	# Print usage message and exit
	30	def usage(*args):
	31	sys.stdout = sys.stderr
	32	for msg in args: print msg
	33	print __doc__
	34	sys.exit(2)
	35
	36	verbose = 1 # 0 for -q, 2 for -v
	37	interactive = 0
	38	mac = 0
	39	rmok = 0
	40	nologin = 0
	41	skippats = ['.', '..', '.mirrorinfo']
	42
	43	# Main program: parse command line and start processing
	44	def main():
	45	global verbose, interactive, mac, rmok, nologin
	46	try:
	47	opts, args = getopt.getopt(sys.argv[1:], 'a:bil:mnp:qrs:v')
	48	except getopt.error, msg:
	49	usage(msg)
	50	login = ''
	51	passwd = ''
	52	account = ''
	53	if not args: usage('hostname missing')
	54	host = args[0]
	55	port = 0
	56	if ':' in host:
	57	host, port = host.split(':', 1)
	58	port = int(port)
	59	try:
	60	auth = netrc.netrc().authenticators(host)
	61	if auth is not None:
	62	login, account, passwd = auth
	63	except (netrc.NetrcParseError, IOError):
	64	pass
	65	for o, a in opts:
	66	if o == '-l': login = a
	67	if o == '-p': passwd = a
	68	if o == '-a': account = a
	69	if o == '-v': verbose = verbose + 1
	70	if o == '-q': verbose = 0
	71	if o == '-i': interactive = 1
	72	if o == '-m': mac = 1; nologin = 1; skippats.append('*.o')
	73	if o == '-n': nologin = 1
	74	if o == '-r': rmok = 1
	75	if o == '-s': skippats.append(a)
	76	remotedir = ''
	77	localdir = ''
	78	if args[1:]:
	79	remotedir = args[1]
	80	if args[2:]:
	81	localdir = args[2]
	82	if args[3:]: usage('too many arguments')
	83	#
	84	f = ftplib.FTP()
	85	if verbose: print "Connecting to '%s%s'..." % (host,
	86	(port and ":%d"%port or ""))
	87	f.connect(host,port)
	88	if not nologin:
	89	if verbose:
	90	print 'Logging in as %r...' % (login or 'anonymous')
	91	f.login(login, passwd, account)
	92	if verbose: print 'OK.'
	93	pwd = f.pwd()
	94	if verbose > 1: print 'PWD =', repr(pwd)
	95	if remotedir:
	96	if verbose > 1: print 'cwd(%s)' % repr(remotedir)
	97	f.cwd(remotedir)
	98	if verbose > 1: print 'OK.'
	99	pwd = f.pwd()
	100	if verbose > 1: print 'PWD =', repr(pwd)
	101	#
	102	mirrorsubdir(f, localdir)
	103
	104	# Core logic: mirror one subdirectory (recursively)
	105	def mirrorsubdir(f, localdir):
	106	pwd = f.pwd()
	107	if localdir and not os.path.isdir(localdir):
	108	if verbose: print 'Creating local directory', repr(localdir)
	109	try:
	110	makedir(localdir)
	111	except os.error, msg:
	112	print "Failed to establish local directory", repr(localdir)
	113	return
	114	infofilename = os.path.join(localdir, '.mirrorinfo')
	115	try:
	116	text = open(infofilename, 'r').read()
	117	except IOError, msg:
	118	text = '{}'
	119	try:
	120	info = eval(text)
	121	except (SyntaxError, NameError):
	122	print 'Bad mirror info in', repr(infofilename)
	123	info = {}
	124	subdirs = []
	125	listing = []
	126	if verbose: print 'Listing remote directory %r...' % (pwd,)
	127	f.retrlines('LIST', listing.append)
	128	filesfound = []
	129	for line in listing:
	130	if verbose > 1: print '-->', repr(line)
	131	if mac:
	132	# Mac listing has just filenames;
	133	# trailing / means subdirectory
	134	filename = line.strip()
	135	mode = '-'
	136	if filename[-1:] == '/':
	137	filename = filename[:-1]
	138	mode = 'd'
	139	infostuff = ''
	140	else:
	141	# Parse, assuming a UNIX listing
	142	words = line.split(None, 8)
	143	if len(words) < 6:
	144	if verbose > 1: print 'Skipping short line'
	145	continue
	146	filename = words[-1].lstrip()
	147	i = filename.find(" -> ")
	148	if i >= 0:
	149	# words[0] had better start with 'l'...
	150	if verbose > 1:
	151	print 'Found symbolic link %r' % (filename,)
	152	linkto = filename[i+4:]
	153	filename = filename[:i]
	154	infostuff = words[-5:-1]
	155	mode = words[0]
	156	skip = 0
	157	for pat in skippats:
	158	if fnmatch(filename, pat):
	159	if verbose > 1:
	160	print 'Skip pattern', repr(pat),
	161	print 'matches', repr(filename)
	162	skip = 1
	163	break
	164	if skip:
	165	continue
	166	if mode[0] == 'd':
	167	if verbose > 1:
	168	print 'Remembering subdirectory', repr(filename)
	169	subdirs.append(filename)
	170	continue
	171	filesfound.append(filename)
	172	if info.has_key(filename) and info[filename] == infostuff:
	173	if verbose > 1:
	174	print 'Already have this version of',repr(filename)
	175	continue
	176	fullname = os.path.join(localdir, filename)
	177	tempname = os.path.join(localdir, '@'+filename)
	178	if interactive:
	179	doit = askabout('file', filename, pwd)
	180	if not doit:
	181	if not info.has_key(filename):
	182	info[filename] = 'Not retrieved'
	183	continue
	184	try:
	185	os.unlink(tempname)
	186	except os.error:
	187	pass
	188	if mode[0] == 'l':
	189	if verbose:
	190	print "Creating symlink %r -> %r" % (filename, linkto)
	191	try:
	192	os.symlink(linkto, tempname)
	193	except IOError, msg:
	194	print "Can't create %r: %s" % (tempname, msg)
	195	continue
	196	else:
	197	try:
	198	fp = open(tempname, 'wb')
	199	except IOError, msg:
	200	print "Can't create %r: %s" % (tempname, msg)
	201	continue
	202	if verbose:
	203	print 'Retrieving %r from %r as %r...' % (filename, pwd, fullname)
	204	if verbose:
	205	fp1 = LoggingFile(fp, 1024, sys.stdout)
	206	else:
	207	fp1 = fp
	208	t0 = time.time()
	209	try:
	210	f.retrbinary('RETR ' + filename,
	211	fp1.write, 8*1024)
	212	except ftplib.error_perm, msg:
	213	print msg
	214	t1 = time.time()
	215	bytes = fp.tell()
	216	fp.close()
	217	if fp1 != fp:
	218	fp1.close()
	219	try:
	220	os.unlink(fullname)
	221	except os.error:
	222	pass # Ignore the error
	223	try:
	224	os.rename(tempname, fullname)
	225	except os.error, msg:
	226	print "Can't rename %r to %r: %s" % (tempname, fullname, msg)
	227	continue
	228	info[filename] = infostuff
	229	writedict(info, infofilename)
	230	if verbose and mode[0] != 'l':
	231	dt = t1 - t0
	232	kbytes = bytes / 1024.0
	233	print int(round(kbytes)),
	234	print 'Kbytes in',
	235	print int(round(dt)),
	236	print 'seconds',
	237	if t1 > t0:
	238	print '(~%d Kbytes/sec)' % \
	239	int(round(kbytes/dt),)
	240	print
	241	#
	242	# Remove files from info that are no longer remote
	243	deletions = 0
	244	for filename in info.keys():
	245	if filename not in filesfound:
	246	if verbose:
	247	print "Removing obsolete info entry for",
	248	print repr(filename), "in", repr(localdir or ".")
	249	del info[filename]
	250	deletions = deletions + 1
	251	if deletions:
	252	writedict(info, infofilename)
	253	#
	254	# Remove local files that are no longer in the remote directory
	255	try:
	256	if not localdir: names = os.listdir(os.curdir)
	257	else: names = os.listdir(localdir)
	258	except os.error:
	259	names = []
	260	for name in names:
	261	if name[0] == '.' or info.has_key(name) or name in subdirs:
	262	continue
	263	skip = 0
	264	for pat in skippats:
	265	if fnmatch(name, pat):
	266	if verbose > 1:
	267	print 'Skip pattern', repr(pat),
	268	print 'matches', repr(name)
	269	skip = 1
	270	break
	271	if skip:
	272	continue
	273	fullname = os.path.join(localdir, name)
	274	if not rmok:
	275	if verbose:
	276	print 'Local file', repr(fullname),
	277	print 'is no longer pertinent'
	278	continue
	279	if verbose: print 'Removing local file/dir', repr(fullname)
	280	remove(fullname)
	281	#
	282	# Recursively mirror subdirectories
	283	for subdir in subdirs:
	284	if interactive:
	285	doit = askabout('subdirectory', subdir, pwd)
	286	if not doit: continue
	287	if verbose: print 'Processing subdirectory', repr(subdir)
	288	localsubdir = os.path.join(localdir, subdir)
	289	pwd = f.pwd()
	290	if verbose > 1:
	291	print 'Remote directory now:', repr(pwd)
	292	print 'Remote cwd', repr(subdir)
	293	try:
	294	f.cwd(subdir)
	295	except ftplib.error_perm, msg:
	296	print "Can't chdir to", repr(subdir), ":", repr(msg)
	297	else:
	298	if verbose: print 'Mirroring as', repr(localsubdir)
	299	mirrorsubdir(f, localsubdir)
	300	if verbose > 1: print 'Remote cwd ..'
	301	f.cwd('..')
	302	newpwd = f.pwd()
	303	if newpwd != pwd:
	304	print 'Ended up in wrong directory after cd + cd ..'
	305	print 'Giving up now.'
	306	break
	307	else:
	308	if verbose > 1: print 'OK.'
	309
	310	# Helper to remove a file or directory tree
	311	def remove(fullname):
	312	if os.path.isdir(fullname) and not os.path.islink(fullname):
	313	try:
	314	names = os.listdir(fullname)
	315	except os.error:
	316	names = []
	317	ok = 1
	318	for name in names:
	319	if not remove(os.path.join(fullname, name)):
	320	ok = 0
	321	if not ok:
	322	return 0
	323	try:
	324	os.rmdir(fullname)
	325	except os.error, msg:
	326	print "Can't remove local directory %r: %s" % (fullname, msg)
	327	return 0
	328	else:
	329	try:
	330	os.unlink(fullname)
	331	except os.error, msg:
	332	print "Can't remove local file %r: %s" % (fullname, msg)
	333	return 0
	334	return 1
	335
	336	# Wrapper around a file for writing to write a hash sign every block.
	337	class LoggingFile:
	338	def __init__(self, fp, blocksize, outfp):
	339	self.fp = fp
	340	self.bytes = 0
	341	self.hashes = 0
	342	self.blocksize = blocksize
	343	self.outfp = outfp
	344	def write(self, data):
	345	self.bytes = self.bytes + len(data)
	346	hashes = int(self.bytes) / self.blocksize
	347	while hashes > self.hashes:
	348	self.outfp.write('#')
	349	self.outfp.flush()
	350	self.hashes = self.hashes + 1
	351	self.fp.write(data)
	352	def close(self):
	353	self.outfp.write('\n')
	354
	355	# Ask permission to download a file.
	356	def askabout(filetype, filename, pwd):
	357	prompt = 'Retrieve %s %s from %s ? [ny] ' % (filetype, filename, pwd)
	358	while 1:
	359	reply = raw_input(prompt).strip().lower()
	360	if reply in ['y', 'ye', 'yes']:
	361	return 1
	362	if reply in ['', 'n', 'no', 'nop', 'nope']:
	363	return 0
	364	print 'Please answer yes or no.'
	365
	366	# Create a directory if it doesn't exist. Recursively create the
	367	# parent directory as well if needed.
	368	def makedir(pathname):
	369	if os.path.isdir(pathname):
	370	return
	371	dirname = os.path.dirname(pathname)
	372	if dirname: makedir(dirname)
	373	os.mkdir(pathname, 0777)
	374
	375	# Write a dictionary to a file in a way that can be read back using
	376	# rval() but is still somewhat readable (i.e. not a single long line).
	377	# Also creates a backup file.
	378	def writedict(dict, filename):
	379	dir, fname = os.path.split(filename)
	380	tempname = os.path.join(dir, '@' + fname)
	381	backup = os.path.join(dir, fname + '~')
	382	try:
	383	os.unlink(backup)
	384	except os.error:
	385	pass
	386	fp = open(tempname, 'w')
	387	fp.write('{\n')
	388	for key, value in dict.items():
	389	fp.write('%r: %r,\n' % (key, value))
	390	fp.write('}\n')
	391	fp.close()
	392	try:
	393	os.rename(filename, backup)
	394	except os.error:
	395	pass
	396	os.rename(tempname, filename)
	397
	398
	399	if __name__ == '__main__':
	400	main()

sys/src/cmd/python/Tools/scripts/google.py+23−0

@@ -0,0 +1,23 @@
	1	#! /usr/bin/env python
	2
	3	import sys, webbrowser
	4
	5	def main():
	6	args = sys.argv[1:]
	7	if not args:
	8	print "Usage: %s querystring" % sys.argv[0]
	9	return
	10	list = []
	11	for arg in args:
	12	if '+' in arg:
	13	arg = arg.replace('+', '%2B')
	14	if ' ' in arg:
	15	arg = '"%s"' % arg
	16	arg = arg.replace(' ', '+')
	17	list.append(arg)
	18	s = '+'.join(list)
	19	url = "http://www.google.com/search?q=%s" % s
	20	webbrowser.open(url)
	21
	22	if __name__ == '__main__':
	23	main()

sys/src/cmd/python/Tools/scripts/gprof2html.py+79−0

@@ -0,0 +1,79 @@
	1	#! /usr/bin/env python2.3
	2
	3	"""Transform gprof(1) output into useful HTML."""
	4
	5	import re, os, sys, cgi, webbrowser
	6
	7	header = """\
	8	<html>
	9	<head>
	10	<title>gprof output (%s)</title>
	11	</head>
	12	<body>
	13	<pre>
	14	"""
	15
	16	trailer = """\
	17	</pre>
	18	</body>
	19	</html>
	20	"""
	21
	22	def add_escapes(input):
	23	for line in input:
	24	yield cgi.escape(line)
	25
	26	def main():
	27	filename = "gprof.out"
	28	if sys.argv[1:]:
	29	filename = sys.argv[1]
	30	outputfilename = filename + ".html"
	31	input = add_escapes(file(filename))
	32	output = file(outputfilename, "w")
	33	output.write(header % filename)
	34	for line in input:
	35	output.write(line)
	36	if line.startswith(" time"):
	37	break
	38	labels = {}
	39	for line in input:
	40	m = re.match(r"(.* )(\w+)\n", line)
	41	if not m:
	42	output.write(line)
	43	break
	44	stuff, fname = m.group(1, 2)
	45	labels[fname] = fname
	46	output.write('%s<a name="flat:%s" href="#call:%s">%s</a>\n' %
	47	(stuff, fname, fname, fname))
	48	for line in input:
	49	output.write(line)
	50	if line.startswith("index % time"):
	51	break
	52	for line in input:
	53	m = re.match(r"(.* )(\w+)(( <cycle.*>)? \[\d+\])\n", line)
	54	if not m:
	55	output.write(line)
	56	if line.startswith("Index by function name"):
	57	break
	58	continue
	59	prefix, fname, suffix = m.group(1, 2, 3)
	60	if fname not in labels:
	61	output.write(line)
	62	continue
	63	if line.startswith("["):
	64	output.write('%s<a name="call:%s" href="#flat:%s">%s</a>%s\n' %
	65	(prefix, fname, fname, fname, suffix))
	66	else:
	67	output.write('%s<a href="#call:%s">%s</a>%s\n' %
	68	(prefix, fname, fname, suffix))
	69	for line in input:
	70	for part in re.findall(r"(\w+(?:\.c)?\|\W+)", line):
	71	if part in labels:
	72	part = '<a href="#call:%s">%s</a>' % (part, part)
	73	output.write(part)
	74	output.write(trailer)
	75	output.close()
	76	webbrowser.open("file:" + os.path.abspath(outputfilename))
	77
	78	if __name__ == '__main__':
	79	main()

sys/src/cmd/python/Tools/scripts/h2py.py+175−0

@@ -0,0 +1,175 @@
	1	#! /usr/bin/env python
	2
	3	# Read #define's and translate to Python code.
	4	# Handle #include statements.
	5	# Handle #define macros with one argument.
	6	# Anything that isn't recognized or doesn't translate into valid
	7	# Python is ignored.
	8
	9	# Without filename arguments, acts as a filter.
	10	# If one or more filenames are given, output is written to corresponding
	11	# filenames in the local directory, translated to all uppercase, with
	12	# the extension replaced by ".py".
	13
	14	# By passing one or more options of the form "-i regular_expression"
	15	# you can specify additional strings to be ignored. This is useful
	16	# e.g. to ignore casts to u_long: simply specify "-i '(u_long)'".
	17
	18	# XXX To do:
	19	# - turn trailing C comments into Python comments
	20	# - turn C Boolean operators "&& \|\| !" into Python "and or not"
	21	# - what to do about #if(def)?
	22	# - what to do about macros with multiple parameters?
	23
	24	import sys, re, getopt, os
	25
	26	p_define = re.compile('^[\t ]#[\t ]define[\t ]+([a-zA-Z0-9_]+)[\t ]+')
	27
	28	p_macro = re.compile(
	29	'^[\t ]#[\t ]define[\t ]+'
	30	'([a-zA-Z0-9_]+)$([_a-zA-Z][_a-zA-Z0-9]*)$[\t ]+')
	31
	32	p_include = re.compile('^[\t ]#[\t ]include[\t ]+<([a-zA-Z0-9_/\.]+)')
	33
	34	p_comment = re.compile(r'/\([^]+\|\+[^/])(\*+/)?')
	35	p_cpp_comment = re.compile('//.*')
	36
	37	ignores = [p_comment, p_cpp_comment]
	38
	39	p_char = re.compile(r"'(\\.[^\\]*\|[^\\])'")
	40
	41	p_hex = re.compile(r"0x([0-9a-fA-F]+)L?")
	42
	43	filedict = {}
	44	importable = {}
	45
	46	try:
	47	searchdirs=os.environ['include'].split(';')
	48	except KeyError:
	49	try:
	50	searchdirs=os.environ['INCLUDE'].split(';')
	51	except KeyError:
	52	try:
	53	if sys.platform.find("beos") == 0:
	54	searchdirs=os.environ['BEINCLUDES'].split(';')
	55	elif sys.platform.startswith("atheos"):
	56	searchdirs=os.environ['C_INCLUDE_PATH'].split(':')
	57	else:
	58	raise KeyError
	59	except KeyError:
	60	searchdirs=['/usr/include']
	61
	62	def main():
	63	global filedict
	64	opts, args = getopt.getopt(sys.argv[1:], 'i:')
	65	for o, a in opts:
	66	if o == '-i':
	67	ignores.append(re.compile(a))
	68	if not args:
	69	args = ['-']
	70	for filename in args:
	71	if filename == '-':
	72	sys.stdout.write('# Generated by h2py from stdin\n')
	73	process(sys.stdin, sys.stdout)
	74	else:
	75	fp = open(filename, 'r')
	76	outfile = os.path.basename(filename)
	77	i = outfile.rfind('.')
	78	if i > 0: outfile = outfile[:i]
	79	modname = outfile.upper()
	80	outfile = modname + '.py'
	81	outfp = open(outfile, 'w')
	82	outfp.write('# Generated by h2py from %s\n' % filename)
	83	filedict = {}
	84	for dir in searchdirs:
	85	if filename[:len(dir)] == dir:
	86	filedict[filename[len(dir)+1:]] = None # no '/' trailing
	87	importable[filename[len(dir)+1:]] = modname
	88	break
	89	process(fp, outfp)
	90	outfp.close()
	91	fp.close()
	92
	93	def pytify(body):
	94	# replace ignored patterns by spaces
	95	for p in ignores:
	96	body = p.sub(' ', body)
	97	# replace char literals by ord(...)
	98	body = p_char.sub('ord(\\0)', body)
	99	# Compute negative hexadecimal constants
	100	start = 0
	101	UMAX = 2*(sys.maxint+1)
	102	while 1:
	103	m = p_hex.search(body, start)
	104	if not m: break
	105	s,e = m.span()
	106	val = long(body[slice(*m.span(1))], 16)
	107	if val > sys.maxint:
	108	val -= UMAX
	109	body = body[:s] + "(" + str(val) + ")" + body[e:]
	110	start = s + 1
	111	return body
	112
	113	def process(fp, outfp, env = {}):
	114	lineno = 0
	115	while 1:
	116	line = fp.readline()
	117	if not line: break
	118	lineno = lineno + 1
	119	match = p_define.match(line)
	120	if match:
	121	# gobble up continuation lines
	122	while line[-2:] == '\\\n':
	123	nextline = fp.readline()
	124	if not nextline: break
	125	lineno = lineno + 1
	126	line = line + nextline
	127	name = match.group(1)
	128	body = line[match.end():]
	129	body = pytify(body)
	130	ok = 0
	131	stmt = '%s = %s\n' % (name, body.strip())
	132	try:
	133	exec stmt in env
	134	except:
	135	sys.stderr.write('Skipping: %s' % stmt)
	136	else:
	137	outfp.write(stmt)
	138	match = p_macro.match(line)
	139	if match:
	140	macro, arg = match.group(1, 2)
	141	body = line[match.end():]
	142	body = pytify(body)
	143	stmt = 'def %s(%s): return %s\n' % (macro, arg, body)
	144	try:
	145	exec stmt in env
	146	except:
	147	sys.stderr.write('Skipping: %s' % stmt)
	148	else:
	149	outfp.write(stmt)
	150	match = p_include.match(line)
	151	if match:
	152	regs = match.regs
	153	a, b = regs[1]
	154	filename = line[a:b]
	155	if importable.has_key(filename):
	156	outfp.write('from %s import *\n' % importable[filename])
	157	elif not filedict.has_key(filename):
	158	filedict[filename] = None
	159	inclfp = None
	160	for dir in searchdirs:
	161	try:
	162	inclfp = open(dir + '/' + filename)
	163	break
	164	except IOError:
	165	pass
	166	if inclfp:
	167	outfp.write(
	168	'\n# Included from %s\n' % filename)
	169	process(inclfp, outfp, env)
	170	else:
	171	sys.stderr.write('Warning - could not find file %s\n' %
	172	filename)
	173
	174	if __name__ == '__main__':
	175	main()

sys/src/cmd/python/Tools/scripts/hotshotmain.py+55−0

@@ -0,0 +1,55 @@
	1	#!/usr/bin/env python
	2	# -- coding: iso-8859-1 --
	3
	4	"""
	5	Run a Python script under hotshot's control.
	6
	7	Adapted from a posting on python-dev by Walter D�rwald
	8
	9	usage %prog [ %prog args ] filename [ filename args ]
	10
	11	Any arguments after the filename are used as sys.argv for the filename.
	12	"""
	13
	14	import sys
	15	import optparse
	16	import os
	17	import hotshot
	18	import hotshot.stats
	19
	20	PROFILE = "hotshot.prof"
	21
	22	def run_hotshot(filename, profile, args):
	23	prof = hotshot.Profile(profile)
	24	sys.path.insert(0, os.path.dirname(filename))
	25	sys.argv = [filename] + args
	26	prof.run("execfile(%r)" % filename)
	27	prof.close()
	28	stats = hotshot.stats.load(profile)
	29	stats.sort_stats("time", "calls")
	30
	31	# print_stats uses unadorned print statements, so the only way
	32	# to force output to stderr is to reassign sys.stdout temporarily
	33	save_stdout = sys.stdout
	34	sys.stdout = sys.stderr
	35	stats.print_stats()
	36	sys.stdout = save_stdout
	37
	38	return 0
	39
	40	def main(args):
	41	parser = optparse.OptionParser(__doc__)
	42	parser.disable_interspersed_args()
	43	parser.add_option("-p", "--profile", action="store", default=PROFILE,
	44	dest="profile", help='Specify profile file to use')
	45	(options, args) = parser.parse_args(args)
	46
	47	if len(args) == 0:
	48	parser.print_help("missing script to execute")
	49	return 1
	50
	51	filename = args[0]
	52	return run_hotshot(filename, options.profile, args[1:])
	53
	54	if __name__ == "__main__":
	55	sys.exit(main(sys.argv[1:]))

sys/src/cmd/python/Tools/scripts/idle+5−0

@@ -0,0 +1,5 @@
	1	#! /usr/bin/env python
	2
	3	from idlelib.PyShell import main
	4	if __name__ == '__main__':
	5	main()

sys/src/cmd/python/Tools/scripts/ifdef.py+112−0

@@ -0,0 +1,112 @@
	1	#! /usr/bin/env python
	2
	3	# Selectively preprocess #ifdef / #ifndef statements.
	4	# Usage:
	5	# ifdef [-Dname] ... [-Uname] ... [file] ...
	6	#
	7	# This scans the file(s), looking for #ifdef and #ifndef preprocessor
	8	# commands that test for one of the names mentioned in the -D and -U
	9	# options. On standard output it writes a copy of the input file(s)
	10	# minus those code sections that are suppressed by the selected
	11	# combination of defined/undefined symbols. The #if(n)def/#else/#else
	12	# lines themselfs (if the #if(n)def tests for one of the mentioned
	13	# names) are removed as well.
	14
	15	# Features: Arbitrary nesting of recognized and unrecognized
	16	# preprocesor statements works correctly. Unrecognized #if* commands
	17	# are left in place, so it will never remove too much, only too
	18	# little. It does accept whitespace around the '#' character.
	19
	20	# Restrictions: There should be no comments or other symbols on the
	21	# #if(n)def lines. The effect of #define/#undef commands in the input
	22	# file or in included files is not taken into account. Tests using
	23	# #if and the defined() pseudo function are not recognized. The #elif
	24	# command is not recognized. Improperly nesting is not detected.
	25	# Lines that look like preprocessor commands but which are actually
	26	# part of comments or string literals will be mistaken for
	27	# preprocessor commands.
	28
	29	import sys
	30	import getopt
	31
	32	defs = []
	33	undefs = []
	34
	35	def main():
	36	opts, args = getopt.getopt(sys.argv[1:], 'D:U:')
	37	for o, a in opts:
	38	if o == '-D':
	39	defs.append(a)
	40	if o == '-U':
	41	undefs.append(a)
	42	if not args:
	43	args = ['-']
	44	for filename in args:
	45	if filename == '-':
	46	process(sys.stdin, sys.stdout)
	47	else:
	48	f = open(filename, 'r')
	49	process(f, sys.stdout)
	50	f.close()
	51
	52	def process(fpi, fpo):
	53	keywords = ('if', 'ifdef', 'ifndef', 'else', 'endif')
	54	ok = 1
	55	stack = []
	56	while 1:
	57	line = fpi.readline()
	58	if not line: break
	59	while line[-2:] == '\\\n':
	60	nextline = fpi.readline()
	61	if not nextline: break
	62	line = line + nextline
	63	tmp = line.strip()
	64	if tmp[:1] != '#':
	65	if ok: fpo.write(line)
	66	continue
	67	tmp = tmp[1:].strip()
	68	words = tmp.split()
	69	keyword = words[0]
	70	if keyword not in keywords:
	71	if ok: fpo.write(line)
	72	continue
	73	if keyword in ('ifdef', 'ifndef') and len(words) == 2:
	74	if keyword == 'ifdef':
	75	ko = 1
	76	else:
	77	ko = 0
	78	word = words[1]
	79	if word in defs:
	80	stack.append((ok, ko, word))
	81	if not ko: ok = 0
	82	elif word in undefs:
	83	stack.append((ok, not ko, word))
	84	if ko: ok = 0
	85	else:
	86	stack.append((ok, -1, word))
	87	if ok: fpo.write(line)
	88	elif keyword == 'if':
	89	stack.append((ok, -1, ''))
	90	if ok: fpo.write(line)
	91	elif keyword == 'else' and stack:
	92	s_ok, s_ko, s_word = stack[-1]
	93	if s_ko < 0:
	94	if ok: fpo.write(line)
	95	else:
	96	s_ko = not s_ko
	97	ok = s_ok
	98	if not s_ko: ok = 0
	99	stack[-1] = s_ok, s_ko, s_word
	100	elif keyword == 'endif' and stack:
	101	s_ok, s_ko, s_word = stack[-1]
	102	if s_ko < 0:
	103	if ok: fpo.write(line)
	104	del stack[-1]
	105	ok = s_ok
	106	else:
	107	sys.stderr.write('Unknown keyword %s\n' % keyword)
	108	if stack:
	109	sys.stderr.write('stack: %s\n' % stack)
	110
	111	if __name__ == '__main__':
	112	main()

sys/src/cmd/python/Tools/scripts/lfcr.py+24−0

@@ -0,0 +1,24 @@
	1	#! /usr/bin/env python
	2
	3	"Replace LF with CRLF in argument files. Print names of changed files."
	4
	5	import sys, re, os
	6
	7	def main():
	8	for filename in sys.argv[1:]:
	9	if os.path.isdir(filename):
	10	print filename, "Directory!"
	11	continue
	12	data = open(filename, "rb").read()
	13	if '\0' in data:
	14	print filename, "Binary!"
	15	continue
	16	newdata = re.sub("\r?\n", "\r\n", data)
	17	if newdata != data:
	18	print filename
	19	f = open(filename, "wb")
	20	f.write(newdata)
	21	f.close()
	22
	23	if __name__ == '__main__':
	24	main()

sys/src/cmd/python/Tools/scripts/linktree.py+80−0

@@ -0,0 +1,80 @@
	1	#! /usr/bin/env python
	2
	3	# linktree
	4	#
	5	# Make a copy of a directory tree with symbolic links to all files in the
	6	# original tree.
	7	# All symbolic links go to a special symbolic link at the top, so you
	8	# can easily fix things if the original source tree moves.
	9	# See also "mkreal".
	10	#
	11	# usage: mklinks oldtree newtree
	12
	13	import sys, os
	14
	15	LINK = '.LINK' # Name of special symlink at the top.
	16
	17	debug = 0
	18
	19	def main():
	20	if not 3 <= len(sys.argv) <= 4:
	21	print 'usage:', sys.argv[0], 'oldtree newtree [linkto]'
	22	return 2
	23	oldtree, newtree = sys.argv[1], sys.argv[2]
	24	if len(sys.argv) > 3:
	25	link = sys.argv[3]
	26	link_may_fail = 1
	27	else:
	28	link = LINK
	29	link_may_fail = 0
	30	if not os.path.isdir(oldtree):
	31	print oldtree + ': not a directory'
	32	return 1
	33	try:
	34	os.mkdir(newtree, 0777)
	35	except os.error, msg:
	36	print newtree + ': cannot mkdir:', msg
	37	return 1
	38	linkname = os.path.join(newtree, link)
	39	try:
	40	os.symlink(os.path.join(os.pardir, oldtree), linkname)
	41	except os.error, msg:
	42	if not link_may_fail:
	43	print linkname + ': cannot symlink:', msg
	44	return 1
	45	else:
	46	print linkname + ': warning: cannot symlink:', msg
	47	linknames(oldtree, newtree, link)
	48	return 0
	49
	50	def linknames(old, new, link):
	51	if debug: print 'linknames', (old, new, link)
	52	try:
	53	names = os.listdir(old)
	54	except os.error, msg:
	55	print old + ': warning: cannot listdir:', msg
	56	return
	57	for name in names:
	58	if name not in (os.curdir, os.pardir):
	59	oldname = os.path.join(old, name)
	60	linkname = os.path.join(link, name)
	61	newname = os.path.join(new, name)
	62	if debug > 1: print oldname, newname, linkname
	63	if os.path.isdir(oldname) and \
	64	not os.path.islink(oldname):
	65	try:
	66	os.mkdir(newname, 0777)
	67	ok = 1
	68	except:
	69	print newname + \
	70	': warning: cannot mkdir:', msg
	71	ok = 0
	72	if ok:
	73	linkname = os.path.join(os.pardir,
	74	linkname)
	75	linknames(oldname, newname, linkname)
	76	else:
	77	os.symlink(linkname, newname)
	78
	79	if __name__ == '__main__':
	80	sys.exit(main())

sys/src/cmd/python/Tools/scripts/lll.py+28−0

@@ -0,0 +1,28 @@
	1	#! /usr/bin/env python
	2
	3	# Find symbolic links and show where they point to.
	4	# Arguments are directories to search; default is current directory.
	5	# No recursion.
	6	# (This is a totally different program from "findsymlinks.py"!)
	7
	8	import sys, os
	9
	10	def lll(dirname):
	11	for name in os.listdir(dirname):
	12	if name not in (os.curdir, os.pardir):
	13	full = os.path.join(dirname, name)
	14	if os.path.islink(full):
	15	print name, '->', os.readlink(full)
	16	def main():
	17	args = sys.argv[1:]
	18	if not args: args = [os.curdir]
	19	first = 1
	20	for arg in args:
	21	if len(args) > 1:
	22	if not first: print
	23	first = 0
	24	print arg + ':'
	25	lll(arg)
	26
	27	if __name__ == '__main__':
	28	main()

sys/src/cmd/python/Tools/scripts/logmerge.py+185−0

@@ -0,0 +1,185 @@
	1	#! /usr/bin/env python
	2
	3	"""Consolidate a bunch of CVS or RCS logs read from stdin.
	4
	5	Input should be the output of a CVS or RCS logging command, e.g.
	6
	7	cvs log -rrelease14:
	8
	9	which dumps all log messages from release1.4 upwards (assuming that
	10	release 1.4 was tagged with tag 'release14'). Note the trailing
	11	colon!
	12
	13	This collects all the revision records and outputs them sorted by date
	14	rather than by file, collapsing duplicate revision record, i.e.,
	15	records with the same message for different files.
	16
	17	The -t option causes it to truncate (discard) the last revision log
	18	entry; this is useful when using something like the above cvs log
	19	command, which shows the revisions including the given tag, while you
	20	probably want everything since that tag.
	21
	22	The -r option reverses the output (oldest first; the default is oldest
	23	last).
	24
	25	The -b tag option restricts the output to only checkin messages
	26	belonging to the given branch tag. The form -b HEAD restricts the
	27	output to checkin messages belonging to the CVS head (trunk). (It
	28	produces some output if tag is a non-branch tag, but this output is
	29	not very useful.)
	30
	31	-h prints this message and exits.
	32
	33	XXX This code was created by reverse engineering CVS 1.9 and RCS 5.7
	34	from their output.
	35	"""
	36
	37	import os, sys, errno, getopt, re
	38
	39	sep1 = '='*77 + '\n' # file separator
	40	sep2 = '-'*28 + '\n' # revision separator
	41
	42	def main():
	43	"""Main program"""
	44	truncate_last = 0
	45	reverse = 0
	46	branch = None
	47	opts, args = getopt.getopt(sys.argv[1:], "trb:h")
	48	for o, a in opts:
	49	if o == '-t':
	50	truncate_last = 1
	51	elif o == '-r':
	52	reverse = 1
	53	elif o == '-b':
	54	branch = a
	55	elif o == '-h':
	56	print __doc__
	57	sys.exit(0)
	58	database = []
	59	while 1:
	60	chunk = read_chunk(sys.stdin)
	61	if not chunk:
	62	break
	63	records = digest_chunk(chunk, branch)
	64	if truncate_last:
	65	del records[-1]
	66	database[len(database):] = records
	67	database.sort()
	68	if not reverse:
	69	database.reverse()
	70	format_output(database)
	71
	72	def read_chunk(fp):
	73	"""Read a chunk -- data for one file, ending with sep1.
	74
	75	Split the chunk in parts separated by sep2.
	76
	77	"""
	78	chunk = []
	79	lines = []
	80	while 1:
	81	line = fp.readline()
	82	if not line:
	83	break
	84	if line == sep1:
	85	if lines:
	86	chunk.append(lines)
	87	break
	88	if line == sep2:
	89	if lines:
	90	chunk.append(lines)
	91	lines = []
	92	else:
	93	lines.append(line)
	94	return chunk
	95
	96	def digest_chunk(chunk, branch=None):
	97	"""Digest a chunk -- extract working file name and revisions"""
	98	lines = chunk[0]
	99	key = 'Working file:'
	100	keylen = len(key)
	101	for line in lines:
	102	if line[:keylen] == key:
	103	working_file = line[keylen:].strip()
	104	break
	105	else:
	106	working_file = None
	107	if branch is None:
	108	pass
	109	elif branch == "HEAD":
	110	branch = re.compile(r"^\d+\.\d+$")
	111	else:
	112	revisions = {}
	113	key = 'symbolic names:\n'
	114	found = 0
	115	for line in lines:
	116	if line == key:
	117	found = 1
	118	elif found:
	119	if line[0] in '\t ':
	120	tag, rev = line.split()
	121	if tag[-1] == ':':
	122	tag = tag[:-1]
	123	revisions[tag] = rev
	124	else:
	125	found = 0
	126	rev = revisions.get(branch)
	127	branch = re.compile(r"^<>$") # <> to force a mismatch by default
	128	if rev:
	129	if rev.find('.0.') >= 0:
	130	rev = rev.replace('.0.', '.')
	131	branch = re.compile(r"^" + re.escape(rev) + r"\.\d+$")
	132	records = []
	133	for lines in chunk[1:]:
	134	revline = lines[0]
	135	dateline = lines[1]
	136	text = lines[2:]
	137	words = dateline.split()
	138	author = None
	139	if len(words) >= 3 and words[0] == 'date:':
	140	dateword = words[1]
	141	timeword = words[2]
	142	if timeword[-1:] == ';':
	143	timeword = timeword[:-1]
	144	date = dateword + ' ' + timeword
	145	if len(words) >= 5 and words[3] == 'author:':
	146	author = words[4]
	147	if author[-1:] == ';':
	148	author = author[:-1]
	149	else:
	150	date = None
	151	text.insert(0, revline)
	152	words = revline.split()
	153	if len(words) >= 2 and words[0] == 'revision':
	154	rev = words[1]
	155	else:
	156	# No 'revision' line -- weird...
	157	rev = None
	158	text.insert(0, revline)
	159	if branch:
	160	if rev is None or not branch.match(rev):
	161	continue
	162	records.append((date, working_file, rev, author, text))
	163	return records
	164
	165	def format_output(database):
	166	prevtext = None
	167	prev = []
	168	database.append((None, None, None, None, None)) # Sentinel
	169	for (date, working_file, rev, author, text) in database:
	170	if text != prevtext:
	171	if prev:
	172	print sep2,
	173	for (p_date, p_working_file, p_rev, p_author) in prev:
	174	print p_date, p_author, p_working_file, p_rev
	175	sys.stdout.writelines(prevtext)
	176	prev = []
	177	prev.append((date, working_file, rev, author))
	178	prevtext = text
	179
	180	if __name__ == '__main__':
	181	try:
	182	main()
	183	except IOError, e:
	184	if e.errno != errno.EPIPE:
	185	raise

sys/src/cmd/python/Tools/scripts/mailerdaemon.py+237−0

@@ -0,0 +1,237 @@
	1	"""mailerdaemon - classes to parse mailer-daemon messages"""
	2
	3	import rfc822
	4	import calendar
	5	import re
	6	import os
	7	import sys
	8
	9	Unparseable = 'mailerdaemon.Unparseable'
	10
	11	class ErrorMessage(rfc822.Message):
	12	def __init__(self, fp):
	13	rfc822.Message.__init__(self, fp)
	14	self.sub = ''
	15
	16	def is_warning(self):
	17	sub = self.getheader('Subject')
	18	if not sub:
	19	return 0
	20	sub = sub.lower()
	21	if sub.startswith('waiting mail'): return 1
	22	if 'warning' in sub: return 1
	23	self.sub = sub
	24	return 0
	25
	26	def get_errors(self):
	27	for p in EMPARSERS:
	28	self.rewindbody()
	29	try:
	30	return p(self.fp, self.sub)
	31	except Unparseable:
	32	pass
	33	raise Unparseable
	34
	35	# List of re's or tuples of re's.
	36	# If a re, it should contain at least a group (?P<email>...) which
	37	# should refer to the email address. The re can also contain a group
	38	# (?P<reason>...) which should refer to the reason (error message).
	39	# If no reason is present, the emparse_list_reason list is used to
	40	# find a reason.
	41	# If a tuple, the tuple should contain 2 re's. The first re finds a
	42	# location, the second re is repeated one or more times to find
	43	# multiple email addresses. The second re is matched (not searched)
	44	# where the previous match ended.
	45	# The re's are compiled using the re module.
	46	emparse_list_list = [
	47	'error: (?P<reason>unresolvable): (?P<email>.+)',
	48	('----- The following addresses had permanent fatal errors -----\n',
	49	'(?P<email>[^ \n].)\n( .\n)?'),
	50	'remote execution.\n.rmail (?P<email>.+)',
	51	('The following recipients did not receive your message:\n\n',
	52	' +(?P<email>.*)\n(The following recipients did not receive your message:\n\n)?'),
	53	'------- Failure Reasons --------\n\n(?P<reason>.)\n(?P<email>.)',
	54	'^<(?P<email>.)>:\n(?P<reason>.)',
	55	'^(?P<reason>User mailbox exceeds allowed size): (?P<email>.+)',
	56	'^5\\d{2} <(?P<email>[^\n>]+)>\\.\\.\\. (?P<reason>.+)',
	57	'^Original-Recipient: rfc822;(?P<email>.*)',
	58	'^did not reach the following recipient\$s\$:\n\n(?P<email>.) on .\n +(?P<reason>.*)',
	59	'^ <(?P<email>[^\n>]+)> \\.\\.\\. (?P<reason>.*)',
	60	'^Report on your message to: (?P<email>.)\nReason: (?P<reason>.)',
	61	'^Your message was not delivered to +(?P<email>.)\n +for the following reason:\n +(?P<reason>.)',
	62	'^ was not +(?P<email>[^ \n].?) \n.\n.\n.\n because:.\n +(?P<reason>[^ \n].?) \n',
	63	]
	64	# compile the re's in the list and store them in-place.
	65	for i in range(len(emparse_list_list)):
	66	x = emparse_list_list[i]
	67	if type(x) is type(''):
	68	x = re.compile(x, re.MULTILINE)
	69	else:
	70	xl = []
	71	for x in x:
	72	xl.append(re.compile(x, re.MULTILINE))
	73	x = tuple(xl)
	74	del xl
	75	emparse_list_list[i] = x
	76	del x
	77	del i
	78
	79	# list of re's used to find reasons (error messages).
	80	# if a string, "<>" is replaced by a copy of the email address.
	81	# The expressions are searched for in order. After the first match,
	82	# no more expressions are searched for. So, order is important.
	83	emparse_list_reason = [
	84	r'^5\d{2} <>\.\.\. (?P<reason>.*)',
	85	'<>\.\.\. (?P<reason>.*)',
	86	re.compile(r'^<<< 5\d{2} (?P<reason>.*)', re.MULTILINE),
	87	re.compile('===== stderr was =====\nrmail: (?P<reason>.*)'),
	88	re.compile('^Diagnostic-Code: (?P<reason>.*)', re.MULTILINE),
	89	]
	90	emparse_list_from = re.compile('^From:', re.IGNORECASE\|re.MULTILINE)
	91	def emparse_list(fp, sub):
	92	data = fp.read()
	93	res = emparse_list_from.search(data)
	94	if res is None:
	95	from_index = len(data)
	96	else:
	97	from_index = res.start(0)
	98	errors = []
	99	emails = []
	100	reason = None
	101	for regexp in emparse_list_list:
	102	if type(regexp) is type(()):
	103	res = regexp[0].search(data, 0, from_index)
	104	if res is not None:
	105	try:
	106	reason = res.group('reason')
	107	except IndexError:
	108	pass
	109	while 1:
	110	res = regexp[1].match(data, res.end(0), from_index)
	111	if res is None:
	112	break
	113	emails.append(res.group('email'))
	114	break
	115	else:
	116	res = regexp.search(data, 0, from_index)
	117	if res is not None:
	118	emails.append(res.group('email'))
	119	try:
	120	reason = res.group('reason')
	121	except IndexError:
	122	pass
	123	break
	124	if not emails:
	125	raise Unparseable
	126	if not reason:
	127	reason = sub
	128	if reason[:15] == 'returned mail: ':
	129	reason = reason[15:]
	130	for regexp in emparse_list_reason:
	131	if type(regexp) is type(''):
	132	for i in range(len(emails)-1,-1,-1):
	133	email = emails[i]
	134	exp = re.compile(re.escape(email).join(regexp.split('<>')), re.MULTILINE)
	135	res = exp.search(data)
	136	if res is not None:
	137	errors.append(' '.join((email.strip()+': '+res.group('reason')).split()))
	138	del emails[i]
	139	continue
	140	res = regexp.search(data)
	141	if res is not None:
	142	reason = res.group('reason')
	143	break
	144	for email in emails:
	145	errors.append(' '.join((email.strip()+': '+reason).split()))
	146	return errors
	147
	148	EMPARSERS = [emparse_list, ]
	149
	150	def sort_numeric(a, b):
	151	a = int(a)
	152	b = int(b)
	153	if a < b: return -1
	154	elif a > b: return 1
	155	else: return 0
	156
	157	def parsedir(dir, modify):
	158	os.chdir(dir)
	159	pat = re.compile('^[0-9]*$')
	160	errordict = {}
	161	errorfirst = {}
	162	errorlast = {}
	163	nok = nwarn = nbad = 0
	164
	165	# find all numeric file names and sort them
	166	files = filter(lambda fn, pat=pat: pat.match(fn) is not None, os.listdir('.'))
	167	files.sort(sort_numeric)
	168
	169	for fn in files:
	170	# Lets try to parse the file.
	171	fp = open(fn)
	172	m = ErrorMessage(fp)
	173	sender = m.getaddr('From')
	174	print '%s\t%-40s\t'%(fn, sender[1]),
	175
	176	if m.is_warning():
	177	fp.close()
	178	print 'warning only'
	179	nwarn = nwarn + 1
	180	if modify:
	181	os.rename(fn, ','+fn)
	182	## os.unlink(fn)
	183	continue
	184
	185	try:
	186	errors = m.get_errors()
	187	except Unparseable:
	188	print '** Not parseable'
	189	nbad = nbad + 1
	190	fp.close()
	191	continue
	192	print len(errors), 'errors'
	193
	194	# Remember them
	195	for e in errors:
	196	try:
	197	mm, dd = m.getdate('date')[1:1+2]
	198	date = '%s %02d' % (calendar.month_abbr[mm], dd)
	199	except:
	200	date = '??????'
	201	if not errordict.has_key(e):
	202	errordict[e] = 1
	203	errorfirst[e] = '%s (%s)' % (fn, date)
	204	else:
	205	errordict[e] = errordict[e] + 1
	206	errorlast[e] = '%s (%s)' % (fn, date)
	207
	208	fp.close()
	209	nok = nok + 1
	210	if modify:
	211	os.rename(fn, ','+fn)
	212	## os.unlink(fn)
	213
	214	print '--------------'
	215	print nok, 'files parsed,',nwarn,'files warning-only,',
	216	print nbad,'files unparseable'
	217	print '--------------'
	218	list = []
	219	for e in errordict.keys():
	220	list.append((errordict[e], errorfirst[e], errorlast[e], e))
	221	list.sort()
	222	for num, first, last, e in list:
	223	print '%d %s - %s\t%s' % (num, first, last, e)
	224
	225	def main():
	226	modify = 0
	227	if len(sys.argv) > 1 and sys.argv[1] == '-d':
	228	modify = 1
	229	del sys.argv[1]
	230	if len(sys.argv) > 1:
	231	for folder in sys.argv[1:]:
	232	parsedir(folder, modify)
	233	else:
	234	parsedir('/ufs/jack/Mail/errorsinbox', modify)
	235
	236	if __name__ == '__main__' or sys.argv[0] == __name__:
	237	main()

sys/src/cmd/python/Tools/scripts/md5sum.py+90−0

@@ -0,0 +1,90 @@
	1	#! /usr/bin/env python
	2
	3	"""Python utility to print MD5 checksums of argument files.
	4	"""
	5
	6
	7	bufsize = 8096
	8	fnfilter = None
	9	rmode = 'rb'
	10
	11	usage = """
	12	usage: sum5 [-b] [-t] [-l] [-s bufsize] [file ...]
	13	-b : read files in binary mode (default)
	14	-t : read files in text mode (you almost certainly don't want this!)
	15	-l : print last pathname component only
	16	-s bufsize: read buffer size (default %d)
	17	file ... : files to sum; '-' or no files means stdin
	18	""" % bufsize
	19
	20	import sys
	21	import os
	22	import getopt
	23	import md5
	24
	25	def sum(*files):
	26	sts = 0
	27	if files and isinstance(files[-1], file):
	28	out, files = files[-1], files[:-1]
	29	else:
	30	out = sys.stdout
	31	if len(files) == 1 and not isinstance(files[0], str):
	32	files = files[0]
	33	for f in files:
	34	if isinstance(f, str):
	35	if f == '-':
	36	sts = printsumfp(sys.stdin, '<stdin>', out) or sts
	37	else:
	38	sts = printsum(f, out) or sts
	39	else:
	40	sts = sum(f, out) or sts
	41	return sts
	42
	43	def printsum(filename, out=sys.stdout):
	44	try:
	45	fp = open(filename, rmode)
	46	except IOError, msg:
	47	sys.stderr.write('%s: Can\'t open: %s\n' % (filename, msg))
	48	return 1
	49	if fnfilter:
	50	filename = fnfilter(filename)
	51	sts = printsumfp(fp, filename, out)
	52	fp.close()
	53	return sts
	54
	55	def printsumfp(fp, filename, out=sys.stdout):
	56	m = md5.new()
	57	try:
	58	while 1:
	59	data = fp.read(bufsize)
	60	if not data:
	61	break
	62	m.update(data)
	63	except IOError, msg:
	64	sys.stderr.write('%s: I/O error: %s\n' % (filename, msg))
	65	return 1
	66	out.write('%s %s\n' % (m.hexdigest(), filename))
	67	return 0
	68
	69	def main(args = sys.argv[1:], out=sys.stdout):
	70	global fnfilter, rmode, bufsize
	71	try:
	72	opts, args = getopt.getopt(args, 'blts:')
	73	except getopt.error, msg:
	74	sys.stderr.write('%s: %s\n%s' % (sys.argv[0], msg, usage))
	75	return 2
	76	for o, a in opts:
	77	if o == '-l':
	78	fnfilter = os.path.basename
	79	elif o == '-b':
	80	rmode = 'rb'
	81	elif o == '-t':
	82	rmode = 'r'
	83	elif o == '-s':
	84	bufsize = int(a)
	85	if not args:
	86	args = ['-']
	87	return sum(args, out)
	88
	89	if __name__ == '__main__' or __name__ == sys.argv[0]:
	90	sys.exit(main(sys.argv[1:], sys.stdout))

sys/src/cmd/python/Tools/scripts/methfix.py+171−0

@@ -0,0 +1,171 @@
	1	#! /usr/bin/env python
	2
	3	# Fix Python source files to avoid using
	4	# def method(self, (arg1, ..., argn)):
	5	# instead of the more rational
	6	# def method(self, arg1, ..., argn):
	7	#
	8	# Command line arguments are files or directories to be processed.
	9	# Directories are searched recursively for files whose name looks
	10	# like a python module.
	11	# Symbolic links are always ignored (except as explicit directory
	12	# arguments). Of course, the original file is kept as a back-up
	13	# (with a "~" attached to its name).
	14	# It complains about binaries (files containing null bytes)
	15	# and about files that are ostensibly not Python files: if the first
	16	# line starts with '#!' and does not contain the string 'python'.
	17	#
	18	# Changes made are reported to stdout in a diff-like format.
	19	#
	20	# Undoubtedly you can do this using find and sed or perl, but this is
	21	# a nice example of Python code that recurses down a directory tree
	22	# and uses regular expressions. Also note several subtleties like
	23	# preserving the file's mode and avoiding to even write a temp file
	24	# when no changes are needed for a file.
	25	#
	26	# NB: by changing only the function fixline() you can turn this
	27	# into a program for a different change to Python programs...
	28
	29	import sys
	30	import re
	31	import os
	32	from stat import *
	33
	34	err = sys.stderr.write
	35	dbg = err
	36	rep = sys.stdout.write
	37
	38	def main():
	39	bad = 0
	40	if not sys.argv[1:]: # No arguments
	41	err('usage: ' + sys.argv[0] + ' file-or-directory ...\n')
	42	sys.exit(2)
	43	for arg in sys.argv[1:]:
	44	if os.path.isdir(arg):
	45	if recursedown(arg): bad = 1
	46	elif os.path.islink(arg):
	47	err(arg + ': will not process symbolic links\n')
	48	bad = 1
	49	else:
	50	if fix(arg): bad = 1
	51	sys.exit(bad)
	52
	53	ispythonprog = re.compile('^[a-zA-Z0-9_]+\.py$')
	54	def ispython(name):
	55	return ispythonprog.match(name) >= 0
	56
	57	def recursedown(dirname):
	58	dbg('recursedown(%r)\n' % (dirname,))
	59	bad = 0
	60	try:
	61	names = os.listdir(dirname)
	62	except os.error, msg:
	63	err('%s: cannot list directory: %r\n' % (dirname, msg))
	64	return 1
	65	names.sort()
	66	subdirs = []
	67	for name in names:
	68	if name in (os.curdir, os.pardir): continue
	69	fullname = os.path.join(dirname, name)
	70	if os.path.islink(fullname): pass
	71	elif os.path.isdir(fullname):
	72	subdirs.append(fullname)
	73	elif ispython(name):
	74	if fix(fullname): bad = 1
	75	for fullname in subdirs:
	76	if recursedown(fullname): bad = 1
	77	return bad
	78
	79	def fix(filename):
	80	## dbg('fix(%r)\n' % (filename,))
	81	try:
	82	f = open(filename, 'r')
	83	except IOError, msg:
	84	err('%s: cannot open: %r\n' % (filename, msg))
	85	return 1
	86	head, tail = os.path.split(filename)
	87	tempname = os.path.join(head, '@' + tail)
	88	g = None
	89	# If we find a match, we rewind the file and start over but
	90	# now copy everything to a temp file.
	91	lineno = 0
	92	while 1:
	93	line = f.readline()
	94	if not line: break
	95	lineno = lineno + 1
	96	if g is None and '\0' in line:
	97	# Check for binary files
	98	err(filename + ': contains null bytes; not fixed\n')
	99	f.close()
	100	return 1
	101	if lineno == 1 and g is None and line[:2] == '#!':
	102	# Check for non-Python scripts
	103	words = line[2:].split()
	104	if words and re.search('[pP]ython', words[0]) < 0:
	105	msg = filename + ': ' + words[0]
	106	msg = msg + ' script; not fixed\n'
	107	err(msg)
	108	f.close()
	109	return 1
	110	while line[-2:] == '\\\n':
	111	nextline = f.readline()
	112	if not nextline: break
	113	line = line + nextline
	114	lineno = lineno + 1
	115	newline = fixline(line)
	116	if newline != line:
	117	if g is None:
	118	try:
	119	g = open(tempname, 'w')
	120	except IOError, msg:
	121	f.close()
	122	err('%s: cannot create: %r\n' % (tempname, msg))
	123	return 1
	124	f.seek(0)
	125	lineno = 0
	126	rep(filename + ':\n')
	127	continue # restart from the beginning
	128	rep(repr(lineno) + '\n')
	129	rep('< ' + line)
	130	rep('> ' + newline)
	131	if g is not None:
	132	g.write(newline)
	133
	134	# End of file
	135	f.close()
	136	if not g: return 0 # No changes
	137
	138	# Finishing touch -- move files
	139
	140	# First copy the file's mode to the temp file
	141	try:
	142	statbuf = os.stat(filename)
	143	os.chmod(tempname, statbuf[ST_MODE] & 07777)
	144	except os.error, msg:
	145	err('%s: warning: chmod failed (%r)\n' % (tempname, msg))
	146	# Then make a backup of the original file as filename~
	147	try:
	148	os.rename(filename, filename + '~')
	149	except os.error, msg:
	150	err('%s: warning: backup failed (%r)\n' % (filename, msg))
	151	# Now move the temp file to the original file
	152	try:
	153	os.rename(tempname, filename)
	154	except os.error, msg:
	155	err('%s: rename failed (%r)\n' % (filename, msg))
	156	return 1
	157	# Return succes
	158	return 0
	159
	160
	161	fixpat = '^[ \t]+def +[a-zA-Z0-9_]+ ( self , (( (.) )) ) *:'
	162	fixprog = re.compile(fixpat)
	163
	164	def fixline(line):
	165	if fixprog.match(line) >= 0:
	166	(a, b), (c, d) = fixprog.regs[1:3]
	167	line = line[:a] + line[c:d] + line[b:]
	168	return line
	169
	170	if __name__ == '__main__':
	171	main()

sys/src/cmd/python/Tools/scripts/mkreal.py+66−0

@@ -0,0 +1,66 @@
	1	#! /usr/bin/env python
	2
	3	# mkreal
	4	#
	5	# turn a symlink to a directory into a real directory
	6
	7	import sys
	8	import os
	9	from stat import *
	10
	11	join = os.path.join
	12
	13	error = 'mkreal error'
	14
	15	BUFSIZE = 32*1024
	16
	17	def mkrealfile(name):
	18	st = os.stat(name) # Get the mode
	19	mode = S_IMODE(st[ST_MODE])
	20	linkto = os.readlink(name) # Make sure again it's a symlink
	21	f_in = open(name, 'r') # This ensures it's a file
	22	os.unlink(name)
	23	f_out = open(name, 'w')
	24	while 1:
	25	buf = f_in.read(BUFSIZE)
	26	if not buf: break
	27	f_out.write(buf)
	28	del f_out # Flush data to disk before changing mode
	29	os.chmod(name, mode)
	30
	31	def mkrealdir(name):
	32	st = os.stat(name) # Get the mode
	33	mode = S_IMODE(st[ST_MODE])
	34	linkto = os.readlink(name)
	35	files = os.listdir(name)
	36	os.unlink(name)
	37	os.mkdir(name, mode)
	38	os.chmod(name, mode)
	39	linkto = join(os.pardir, linkto)
	40	#
	41	for filename in files:
	42	if filename not in (os.curdir, os.pardir):
	43	os.symlink(join(linkto, filename), join(name, filename))
	44
	45	def main():
	46	sys.stdout = sys.stderr
	47	progname = os.path.basename(sys.argv[0])
	48	if progname == '-c': progname = 'mkreal'
	49	args = sys.argv[1:]
	50	if not args:
	51	print 'usage:', progname, 'path ...'
	52	sys.exit(2)
	53	status = 0
	54	for name in args:
	55	if not os.path.islink(name):
	56	print progname+':', name+':', 'not a symlink'
	57	status = 1
	58	else:
	59	if os.path.isdir(name):
	60	mkrealdir(name)
	61	else:
	62	mkrealfile(name)
	63	sys.exit(status)
	64
	65	if __name__ == '__main__':
	66	main()

sys/src/cmd/python/Tools/scripts/ndiff.py+133−0

@@ -0,0 +1,133 @@
	1	#! /usr/bin/env python
	2
	3	# Module ndiff version 1.7.0
	4	# Released to the public domain 08-Dec-2000,
	5	# by Tim Peters (tim.one@home.com).
	6
	7	# Provided as-is; use at your own risk; no warranty; no promises; enjoy!
	8
	9	# ndiff.py is now simply a front-end to the difflib.ndiff() function.
	10	# Originally, it contained the difflib.SequenceMatcher class as well.
	11	# This completes the raiding of reusable code from this formerly
	12	# self-contained script.
	13
	14	"""ndiff [-q] file1 file2
	15	or
	16	ndiff (-r1 \| -r2) < ndiff_output > file1_or_file2
	17
	18	Print a human-friendly file difference report to stdout. Both inter-
	19	and intra-line differences are noted. In the second form, recreate file1
	20	(-r1) or file2 (-r2) on stdout, from an ndiff report on stdin.
	21
	22	In the first form, if -q ("quiet") is not specified, the first two lines
	23	of output are
	24
	25	-: file1
	26	+: file2
	27
	28	Each remaining line begins with a two-letter code:
	29
	30	"- " line unique to file1
	31	"+ " line unique to file2
	32	" " line common to both files
	33	"? " line not present in either input file
	34
	35	Lines beginning with "? " attempt to guide the eye to intraline
	36	differences, and were not present in either input file. These lines can be
	37	confusing if the source files contain tab characters.
	38
	39	The first file can be recovered by retaining only lines that begin with
	40	" " or "- ", and deleting those 2-character prefixes; use ndiff with -r1.
	41
	42	The second file can be recovered similarly, but by retaining only " " and
	43	"+ " lines; use ndiff with -r2; or, on Unix, the second file can be
	44	recovered by piping the output through
	45
	46	sed -n '/^[+ ] /s/^..//p'
	47	"""
	48
	49	__version__ = 1, 7, 0
	50
	51	import difflib, sys
	52
	53	def fail(msg):
	54	out = sys.stderr.write
	55	out(msg + "\n\n")
	56	out(__doc__)
	57	return 0
	58
	59	# open a file & return the file object; gripe and return 0 if it
	60	# couldn't be opened
	61	def fopen(fname):
	62	try:
	63	return open(fname, 'U')
	64	except IOError, detail:
	65	return fail("couldn't open " + fname + ": " + str(detail))
	66
	67	# open two files & spray the diff to stdout; return false iff a problem
	68	def fcompare(f1name, f2name):
	69	f1 = fopen(f1name)
	70	f2 = fopen(f2name)
	71	if not f1 or not f2:
	72	return 0
	73
	74	a = f1.readlines(); f1.close()
	75	b = f2.readlines(); f2.close()
	76	for line in difflib.ndiff(a, b):
	77	print line,
	78
	79	return 1
	80
	81	# crack args (sys.argv[1:] is normal) & compare;
	82	# return false iff a problem
	83
	84	def main(args):
	85	import getopt
	86	try:
	87	opts, args = getopt.getopt(args, "qr:")
	88	except getopt.error, detail:
	89	return fail(str(detail))
	90	noisy = 1
	91	qseen = rseen = 0
	92	for opt, val in opts:
	93	if opt == "-q":
	94	qseen = 1
	95	noisy = 0
	96	elif opt == "-r":
	97	rseen = 1
	98	whichfile = val
	99	if qseen and rseen:
	100	return fail("can't specify both -q and -r")
	101	if rseen:
	102	if args:
	103	return fail("no args allowed with -r option")
	104	if whichfile in ("1", "2"):
	105	restore(whichfile)
	106	return 1
	107	return fail("-r value must be 1 or 2")
	108	if len(args) != 2:
	109	return fail("need 2 filename args")
	110	f1name, f2name = args
	111	if noisy:
	112	print '-:', f1name
	113	print '+:', f2name
	114	return fcompare(f1name, f2name)
	115
	116	# read ndiff output from stdin, and print file1 (which=='1') or
	117	# file2 (which=='2') to stdout
	118
	119	def restore(which):
	120	restored = difflib.restore(sys.stdin.readlines(), which)
	121	sys.stdout.writelines(restored)
	122
	123	if __name__ == '__main__':
	124	args = sys.argv[1:]
	125	if "-profile" in args:
	126	import profile, pstats
	127	args.remove("-profile")
	128	statf = "ndiff.pro"
	129	profile.run("main(args)", statf)
	130	stats = pstats.Stats(statf)
	131	stats.strip_dirs().sort_stats('time').print_stats()
	132	else:
	133	main(args)

sys/src/cmd/python/Tools/scripts/nm2def.py+103−0

@@ -0,0 +1,103 @@
	1	#! /usr/bin/env python
	2	"""nm2def.py
	3
	4	Helpers to extract symbols from Unix libs and auto-generate
	5	Windows definition files from them. Depends on nm(1). Tested
	6	on Linux and Solaris only (-p option to nm is for Solaris only).
	7
	8	By Marc-Andre Lemburg, Aug 1998.
	9
	10	Additional notes: the output of nm is supposed to look like this:
	11
	12	acceler.o:
	13	000001fd T PyGrammar_AddAccelerators
	14	U PyGrammar_FindDFA
	15	00000237 T PyGrammar_RemoveAccelerators
	16	U _IO_stderr_
	17	U exit
	18	U fprintf
	19	U free
	20	U malloc
	21	U printf
	22
	23	grammar1.o:
	24	00000000 T PyGrammar_FindDFA
	25	00000034 T PyGrammar_LabelRepr
	26	U _PyParser_TokenNames
	27	U abort
	28	U printf
	29	U sprintf
	30
	31	...
	32
	33	Even if this isn't the default output of your nm, there is generally an
	34	option to produce this format (since it is the original v7 Unix format).
	35
	36	"""
	37	import os,re,sys
	38
	39	PYTHONLIB = 'libpython'+sys.version[:3]+'.a'
	40	PC_PYTHONLIB = 'Python'+sys.version[0]+sys.version[2]+'.dll'
	41	NM = 'nm -p -g %s' # For Linux, use "nm -g %s"
	42
	43	def symbols(lib=PYTHONLIB,types=('T','C','D')):
	44
	45	lines = os.popen(NM % lib).readlines()
	46	lines = [s.strip() for s in lines]
	47	symbols = {}
	48	for line in lines:
	49	if len(line) == 0 or ':' in line:
	50	continue
	51	items = line.split()
	52	if len(items) != 3:
	53	continue
	54	address, type, name = items
	55	if type not in types:
	56	continue
	57	symbols[name] = address,type
	58	return symbols
	59
	60	def export_list(symbols):
	61
	62	data = []
	63	code = []
	64	for name,(addr,type) in symbols.items():
	65	if type in ('C','D'):
	66	data.append('\t'+name)
	67	else:
	68	code.append('\t'+name)
	69	data.sort()
	70	data.append('')
	71	code.sort()
	72	return ' DATA\n'.join(data)+'\n'+'\n'.join(code)
	73
	74	# Definition file template
	75	DEF_TEMPLATE = """\
	76	EXPORTS
	77	%s
	78	"""
	79
	80	# Special symbols that have to be included even though they don't
	81	# pass the filter
	82	SPECIALS = (
	83	)
	84
	85	def filter_Python(symbols,specials=SPECIALS):
	86
	87	for name in symbols.keys():
	88	if name[:2] == 'Py' or name[:3] == '_Py':
	89	pass
	90	elif name not in specials:
	91	del symbols[name]
	92
	93	def main():
	94
	95	s = symbols(PYTHONLIB)
	96	filter_Python(s)
	97	exports = export_list(s)
	98	f = sys.stdout # open('PC/python_nt.def','w')
	99	f.write(DEF_TEMPLATE % (exports))
	100	f.close()
	101
	102	if __name__ == '__main__':
	103	main()

sys/src/cmd/python/Tools/scripts/objgraph.py+215−0

@@ -0,0 +1,215 @@
	1	#! /usr/bin/env python
	2
	3	# objgraph
	4	#
	5	# Read "nm -o" input (on IRIX: "nm -Bo") of a set of libraries or modules
	6	# and print various interesting listings, such as:
	7	#
	8	# - which names are used but not defined in the set (and used where),
	9	# - which names are defined in the set (and where),
	10	# - which modules use which other modules,
	11	# - which modules are used by which other modules.
	12	#
	13	# Usage: objgraph [-cdu] [file] ...
	14	# -c: print callers per objectfile
	15	# -d: print callees per objectfile
	16	# -u: print usage of undefined symbols
	17	# If none of -cdu is specified, all are assumed.
	18	# Use "nm -o" to generate the input (on IRIX: "nm -Bo"),
	19	# e.g.: nm -o /lib/libc.a \| objgraph
	20
	21
	22	import sys
	23	import os
	24	import getopt
	25	import re
	26
	27	# Types of symbols.
	28	#
	29	definitions = 'TRGDSBAEC'
	30	externals = 'UV'
	31	ignore = 'Nntrgdsbavuc'
	32
	33	# Regular expression to parse "nm -o" output.
	34	#
	35	matcher = re.compile('(.):\t?........ (.) (.)$')
	36
	37	# Store "item" in "dict" under "key".
	38	# The dictionary maps keys to lists of items.
	39	# If there is no list for the key yet, it is created.
	40	#
	41	def store(dict, key, item):
	42	if dict.has_key(key):
	43	dict[key].append(item)
	44	else:
	45	dict[key] = [item]
	46
	47	# Return a flattened version of a list of strings: the concatenation
	48	# of its elements with intervening spaces.
	49	#
	50	def flat(list):
	51	s = ''
	52	for item in list:
	53	s = s + ' ' + item
	54	return s[1:]
	55
	56	# Global variables mapping defined/undefined names to files and back.
	57	#
	58	file2undef = {}
	59	def2file = {}
	60	file2def = {}
	61	undef2file = {}
	62
	63	# Read one input file and merge the data into the tables.
	64	# Argument is an open file.
	65	#
	66	def readinput(fp):
	67	while 1:
	68	s = fp.readline()
	69	if not s:
	70	break
	71	# If you get any output from this line,
	72	# it is probably caused by an unexpected input line:
	73	if matcher.search(s) < 0: s; continue # Shouldn't happen
	74	(ra, rb), (r1a, r1b), (r2a, r2b), (r3a, r3b) = matcher.regs[:4]
	75	fn, name, type = s[r1a:r1b], s[r3a:r3b], s[r2a:r2b]
	76	if type in definitions:
	77	store(def2file, name, fn)
	78	store(file2def, fn, name)
	79	elif type in externals:
	80	store(file2undef, fn, name)
	81	store(undef2file, name, fn)
	82	elif not type in ignore:
	83	print fn + ':' + name + ': unknown type ' + type
	84
	85	# Print all names that were undefined in some module and where they are
	86	# defined.
	87	#
	88	def printcallee():
	89	flist = file2undef.keys()
	90	flist.sort()
	91	for filename in flist:
	92	print filename + ':'
	93	elist = file2undef[filename]
	94	elist.sort()
	95	for ext in elist:
	96	if len(ext) >= 8:
	97	tabs = '\t'
	98	else:
	99	tabs = '\t\t'
	100	if not def2file.has_key(ext):
	101	print '\t' + ext + tabs + ' *undefined'
	102	else:
	103	print '\t' + ext + tabs + flat(def2file[ext])
	104
	105	# Print for each module the names of the other modules that use it.
	106	#
	107	def printcaller():
	108	files = file2def.keys()
	109	files.sort()
	110	for filename in files:
	111	callers = []
	112	for label in file2def[filename]:
	113	if undef2file.has_key(label):
	114	callers = callers + undef2file[label]
	115	if callers:
	116	callers.sort()
	117	print filename + ':'
	118	lastfn = ''
	119	for fn in callers:
	120	if fn <> lastfn:
	121	print '\t' + fn
	122	lastfn = fn
	123	else:
	124	print filename + ': unused'
	125
	126	# Print undefined names and where they are used.
	127	#
	128	def printundef():
	129	undefs = {}
	130	for filename in file2undef.keys():
	131	for ext in file2undef[filename]:
	132	if not def2file.has_key(ext):
	133	store(undefs, ext, filename)
	134	elist = undefs.keys()
	135	elist.sort()
	136	for ext in elist:
	137	print ext + ':'
	138	flist = undefs[ext]
	139	flist.sort()
	140	for filename in flist:
	141	print '\t' + filename
	142
	143	# Print warning messages about names defined in more than one file.
	144	#
	145	def warndups():
	146	savestdout = sys.stdout
	147	sys.stdout = sys.stderr
	148	names = def2file.keys()
	149	names.sort()
	150	for name in names:
	151	if len(def2file[name]) > 1:
	152	print 'warning:', name, 'multiply defined:',
	153	print flat(def2file[name])
	154	sys.stdout = savestdout
	155
	156	# Main program
	157	#
	158	def main():
	159	try:
	160	optlist, args = getopt.getopt(sys.argv[1:], 'cdu')
	161	except getopt.error:
	162	sys.stdout = sys.stderr
	163	print 'Usage:', os.path.basename(sys.argv[0]),
	164	print '[-cdu] [file] ...'
	165	print '-c: print callers per objectfile'
	166	print '-d: print callees per objectfile'
	167	print '-u: print usage of undefined symbols'
	168	print 'If none of -cdu is specified, all are assumed.'
	169	print 'Use "nm -o" to generate the input (on IRIX: "nm -Bo"),'
	170	print 'e.g.: nm -o /lib/libc.a \| objgraph'
	171	return 1
	172	optu = optc = optd = 0
	173	for opt, void in optlist:
	174	if opt == '-u':
	175	optu = 1
	176	elif opt == '-c':
	177	optc = 1
	178	elif opt == '-d':
	179	optd = 1
	180	if optu == optc == optd == 0:
	181	optu = optc = optd = 1
	182	if not args:
	183	args = ['-']
	184	for filename in args:
	185	if filename == '-':
	186	readinput(sys.stdin)
	187	else:
	188	readinput(open(filename, 'r'))
	189	#
	190	warndups()
	191	#
	192	more = (optu + optc + optd > 1)
	193	if optd:
	194	if more:
	195	print '---------------All callees------------------'
	196	printcallee()
	197	if optu:
	198	if more:
	199	print '---------------Undefined callees------------'
	200	printundef()
	201	if optc:
	202	if more:
	203	print '---------------All Callers------------------'
	204	printcaller()
	205	return 0
	206
	207	# Call the main program.
	208	# Use its return value as exit status.
	209	# Catch interrupts to avoid stack trace.
	210	#
	211	if __name__ == '__main__':
	212	try:
	213	sys.exit(main())
	214	except KeyboardInterrupt:
	215	sys.exit(1)

sys/src/cmd/python/Tools/scripts/parseentities.py+64−0

@@ -0,0 +1,64 @@
	1	#!/usr/local/bin/python
	2	""" Utility for parsing HTML entity definitions available from:
	3
	4	http://www.w3.org/ as e.g.
	5	http://www.w3.org/TR/REC-html40/HTMLlat1.ent
	6
	7	Input is read from stdin, output is written to stdout in form of a
	8	Python snippet defining a dictionary "entitydefs" mapping literal
	9	entity name to character or numeric entity.
	10
	11	Marc-Andre Lemburg, mal@lemburg.com, 1999.
	12	Use as you like. NO WARRANTIES.
	13
	14	"""
	15	import re,sys
	16	import TextTools
	17
	18	entityRE = re.compile('<!ENTITY +(\w+) +CDATA +"([^"]+)" +-- +((?:.\|\n)+?) *-->')
	19
	20	def parse(text,pos=0,endpos=None):
	21
	22	pos = 0
	23	if endpos is None:
	24	endpos = len(text)
	25	d = {}
	26	while 1:
	27	m = entityRE.search(text,pos,endpos)
	28	if not m:
	29	break
	30	name,charcode,comment = m.groups()
	31	d[name] = charcode,comment
	32	pos = m.end()
	33	return d
	34
	35	def writefile(f,defs):
	36
	37	f.write("entitydefs = {\n")
	38	items = defs.items()
	39	items.sort()
	40	for name,(charcode,comment) in items:
	41	if charcode[:2] == '&#':
	42	code = int(charcode[2:-1])
	43	if code < 256:
	44	charcode = "'\%o'" % code
	45	else:
	46	charcode = repr(charcode)
	47	else:
	48	charcode = repr(charcode)
	49	comment = TextTools.collapse(comment)
	50	f.write(" '%s':\t%s, \t# %s\n" % (name,charcode,comment))
	51	f.write('\n}\n')
	52
	53	if __name__ == '__main__':
	54	if len(sys.argv) > 1:
	55	infile = open(sys.argv[1])
	56	else:
	57	infile = sys.stdin
	58	if len(sys.argv) > 2:
	59	outfile = open(sys.argv[2],'w')
	60	else:
	61	outfile = sys.stdout
	62	text = infile.read()
	63	defs = parse(text)
	64	writefile(outfile,defs)

sys/src/cmd/python/Tools/scripts/pathfix.py+149−0

@@ -0,0 +1,149 @@
	1	#! /usr/bin/env python
	2
	3	# Change the #! line occurring in Python scripts. The new interpreter
	4	# pathname must be given with a -i option.
	5	#
	6	# Command line arguments are files or directories to be processed.
	7	# Directories are searched recursively for files whose name looks
	8	# like a python module.
	9	# Symbolic links are always ignored (except as explicit directory
	10	# arguments). Of course, the original file is kept as a back-up
	11	# (with a "~" attached to its name).
	12	#
	13	# Undoubtedly you can do this using find and sed or perl, but this is
	14	# a nice example of Python code that recurses down a directory tree
	15	# and uses regular expressions. Also note several subtleties like
	16	# preserving the file's mode and avoiding to even write a temp file
	17	# when no changes are needed for a file.
	18	#
	19	# NB: by changing only the function fixfile() you can turn this
	20	# into a program for a different change to Python programs...
	21
	22	import sys
	23	import re
	24	import os
	25	from stat import *
	26	import getopt
	27
	28	err = sys.stderr.write
	29	dbg = err
	30	rep = sys.stdout.write
	31
	32	new_interpreter = None
	33
	34	def main():
	35	global new_interpreter
	36	usage = ('usage: %s -i /interpreter file-or-directory ...\n' %
	37	sys.argv[0])
	38	try:
	39	opts, args = getopt.getopt(sys.argv[1:], 'i:')
	40	except getopt.error, msg:
	41	err(msg + '\n')
	42	err(usage)
	43	sys.exit(2)
	44	for o, a in opts:
	45	if o == '-i':
	46	new_interpreter = a
	47	if not new_interpreter or new_interpreter[0] != '/' or not args:
	48	err('-i option or file-or-directory missing\n')
	49	err(usage)
	50	sys.exit(2)
	51	bad = 0
	52	for arg in args:
	53	if os.path.isdir(arg):
	54	if recursedown(arg): bad = 1
	55	elif os.path.islink(arg):
	56	err(arg + ': will not process symbolic links\n')
	57	bad = 1
	58	else:
	59	if fix(arg): bad = 1
	60	sys.exit(bad)
	61
	62	ispythonprog = re.compile('^[a-zA-Z0-9_]+\.py$')
	63	def ispython(name):
	64	return ispythonprog.match(name) >= 0
	65
	66	def recursedown(dirname):
	67	dbg('recursedown(%r)\n' % (dirname,))
	68	bad = 0
	69	try:
	70	names = os.listdir(dirname)
	71	except os.error, msg:
	72	err('%s: cannot list directory: %r\n' % (dirname, msg))
	73	return 1
	74	names.sort()
	75	subdirs = []
	76	for name in names:
	77	if name in (os.curdir, os.pardir): continue
	78	fullname = os.path.join(dirname, name)
	79	if os.path.islink(fullname): pass
	80	elif os.path.isdir(fullname):
	81	subdirs.append(fullname)
	82	elif ispython(name):
	83	if fix(fullname): bad = 1
	84	for fullname in subdirs:
	85	if recursedown(fullname): bad = 1
	86	return bad
	87
	88	def fix(filename):
	89	## dbg('fix(%r)\n' % (filename,))
	90	try:
	91	f = open(filename, 'r')
	92	except IOError, msg:
	93	err('%s: cannot open: %r\n' % (filename, msg))
	94	return 1
	95	line = f.readline()
	96	fixed = fixline(line)
	97	if line == fixed:
	98	rep(filename+': no change\n')
	99	f.close()
	100	return
	101	head, tail = os.path.split(filename)
	102	tempname = os.path.join(head, '@' + tail)
	103	try:
	104	g = open(tempname, 'w')
	105	except IOError, msg:
	106	f.close()
	107	err('%s: cannot create: %r\n' % (tempname, msg))
	108	return 1
	109	rep(filename + ': updating\n')
	110	g.write(fixed)
	111	BUFSIZE = 8*1024
	112	while 1:
	113	buf = f.read(BUFSIZE)
	114	if not buf: break
	115	g.write(buf)
	116	g.close()
	117	f.close()
	118
	119	# Finishing touch -- move files
	120
	121	# First copy the file's mode to the temp file
	122	try:
	123	statbuf = os.stat(filename)
	124	os.chmod(tempname, statbuf[ST_MODE] & 07777)
	125	except os.error, msg:
	126	err('%s: warning: chmod failed (%r)\n' % (tempname, msg))
	127	# Then make a backup of the original file as filename~
	128	try:
	129	os.rename(filename, filename + '~')
	130	except os.error, msg:
	131	err('%s: warning: backup failed (%r)\n' % (filename, msg))
	132	# Now move the temp file to the original file
	133	try:
	134	os.rename(tempname, filename)
	135	except os.error, msg:
	136	err('%s: rename failed (%r)\n' % (filename, msg))
	137	return 1
	138	# Return succes
	139	return 0
	140
	141	def fixline(line):
	142	if not line.startswith('#!'):
	143	return line
	144	if "python" not in line:
	145	return line
	146	return '#! %s\n' % new_interpreter
	147
	148	if __name__ == '__main__':
	149	main()

sys/src/cmd/python/Tools/scripts/pdeps.py+167−0

@@ -0,0 +1,167 @@
	1	#! /usr/bin/env python
	2
	3	# pdeps
	4	#
	5	# Find dependencies between a bunch of Python modules.
	6	#
	7	# Usage:
	8	# pdeps file1.py file2.py ...
	9	#
	10	# Output:
	11	# Four tables separated by lines like '--- Closure ---':
	12	# 1) Direct dependencies, listing which module imports which other modules
	13	# 2) The inverse of (1)
	14	# 3) Indirect dependencies, or the closure of the above
	15	# 4) The inverse of (3)
	16	#
	17	# To do:
	18	# - command line options to select output type
	19	# - option to automatically scan the Python library for referenced modules
	20	# - option to limit output to particular modules
	21
	22
	23	import sys
	24	import re
	25	import os
	26
	27
	28	# Main program
	29	#
	30	def main():
	31	args = sys.argv[1:]
	32	if not args:
	33	print 'usage: pdeps file.py file.py ...'
	34	return 2
	35	#
	36	table = {}
	37	for arg in args:
	38	process(arg, table)
	39	#
	40	print '--- Uses ---'
	41	printresults(table)
	42	#
	43	print '--- Used By ---'
	44	inv = inverse(table)
	45	printresults(inv)
	46	#
	47	print '--- Closure of Uses ---'
	48	reach = closure(table)
	49	printresults(reach)
	50	#
	51	print '--- Closure of Used By ---'
	52	invreach = inverse(reach)
	53	printresults(invreach)
	54	#
	55	return 0
	56
	57
	58	# Compiled regular expressions to search for import statements
	59	#
	60	m_import = re.compile('^[ \t]*from[ \t]+([^ \t]+)[ \t]+')
	61	m_from = re.compile('^[ \t]*import[ \t]+([^#]+)')
	62
	63
	64	# Collect data from one file
	65	#
	66	def process(filename, table):
	67	fp = open(filename, 'r')
	68	mod = os.path.basename(filename)
	69	if mod[-3:] == '.py':
	70	mod = mod[:-3]
	71	table[mod] = list = []
	72	while 1:
	73	line = fp.readline()
	74	if not line: break
	75	while line[-1:] == '\\':
	76	nextline = fp.readline()
	77	if not nextline: break
	78	line = line[:-1] + nextline
	79	if m_import.match(line) >= 0:
	80	(a, b), (a1, b1) = m_import.regs[:2]
	81	elif m_from.match(line) >= 0:
	82	(a, b), (a1, b1) = m_from.regs[:2]
	83	else: continue
	84	words = line[a1:b1].split(',')
	85	# print '#', line, words
	86	for word in words:
	87	word = word.strip()
	88	if word not in list:
	89	list.append(word)
	90
	91
	92	# Compute closure (this is in fact totally general)
	93	#
	94	def closure(table):
	95	modules = table.keys()
	96	#
	97	# Initialize reach with a copy of table
	98	#
	99	reach = {}
	100	for mod in modules:
	101	reach[mod] = table[mod][:]
	102	#
	103	# Iterate until no more change
	104	#
	105	change = 1
	106	while change:
	107	change = 0
	108	for mod in modules:
	109	for mo in reach[mod]:
	110	if mo in modules:
	111	for m in reach[mo]:
	112	if m not in reach[mod]:
	113	reach[mod].append(m)
	114	change = 1
	115	#
	116	return reach
	117
	118
	119	# Invert a table (this is again totally general).
	120	# All keys of the original table are made keys of the inverse,
	121	# so there may be empty lists in the inverse.
	122	#
	123	def inverse(table):
	124	inv = {}
	125	for key in table.keys():
	126	if not inv.has_key(key):
	127	inv[key] = []
	128	for item in table[key]:
	129	store(inv, item, key)
	130	return inv
	131
	132
	133	# Store "item" in "dict" under "key".
	134	# The dictionary maps keys to lists of items.
	135	# If there is no list for the key yet, it is created.
	136	#
	137	def store(dict, key, item):
	138	if dict.has_key(key):
	139	dict[key].append(item)
	140	else:
	141	dict[key] = [item]
	142
	143
	144	# Tabulate results neatly
	145	#
	146	def printresults(table):
	147	modules = table.keys()
	148	maxlen = 0
	149	for mod in modules: maxlen = max(maxlen, len(mod))
	150	modules.sort()
	151	for mod in modules:
	152	list = table[mod]
	153	list.sort()
	154	print mod.ljust(maxlen), ':',
	155	if mod in list:
	156	print '(*)',
	157	for ref in list:
	158	print ref,
	159	print
	160
	161
	162	# Call main and honor exit status
	163	if __name__ == '__main__':
	164	try:
	165	sys.exit(main())
	166	except KeyboardInterrupt:
	167	sys.exit(1)

sys/src/cmd/python/Tools/scripts/pickle2db.py+147−0

@@ -0,0 +1,147 @@
	1	#!/usr/bin/env python
	2
	3	"""
	4	Synopsis: %(prog)s [-h\|-b\|-g\|-r\|-a\|-d] [ picklefile ] dbfile
	5
	6	Read the given picklefile as a series of key/value pairs and write to a new
	7	database. If the database already exists, any contents are deleted. The
	8	optional flags indicate the type of the output database:
	9
	10	-a - open using anydbm
	11	-b - open as bsddb btree file
	12	-d - open as dbm file
	13	-g - open as gdbm file
	14	-h - open as bsddb hash file
	15	-r - open as bsddb recno file
	16
	17	The default is hash. If a pickle file is named it is opened for read
	18	access. If no pickle file is named, the pickle input is read from standard
	19	input.
	20
	21	Note that recno databases can only contain integer keys, so you can't dump a
	22	hash or btree database using db2pickle.py and reconstitute it to a recno
	23	database with %(prog)s unless your keys are integers.
	24
	25	"""
	26
	27	import getopt
	28	try:
	29	import bsddb
	30	except ImportError:
	31	bsddb = None
	32	try:
	33	import dbm
	34	except ImportError:
	35	dbm = None
	36	try:
	37	import gdbm
	38	except ImportError:
	39	gdbm = None
	40	try:
	41	import anydbm
	42	except ImportError:
	43	anydbm = None
	44	import sys
	45	try:
	46	import cPickle as pickle
	47	except ImportError:
	48	import pickle
	49
	50	prog = sys.argv[0]
	51
	52	def usage():
	53	sys.stderr.write(__doc__ % globals())
	54
	55	def main(args):
	56	try:
	57	opts, args = getopt.getopt(args, "hbrdag",
	58	["hash", "btree", "recno", "dbm", "anydbm",
	59	"gdbm"])
	60	except getopt.error:
	61	usage()
	62	return 1
	63
	64	if len(args) == 0 or len(args) > 2:
	65	usage()
	66	return 1
	67	elif len(args) == 1:
	68	pfile = sys.stdin
	69	dbfile = args[0]
	70	else:
	71	try:
	72	pfile = open(args[0], 'rb')
	73	except IOError:
	74	sys.stderr.write("Unable to open %s\n" % args[0])
	75	return 1
	76	dbfile = args[1]
	77
	78	dbopen = None
	79	for opt, arg in opts:
	80	if opt in ("-h", "--hash"):
	81	try:
	82	dbopen = bsddb.hashopen
	83	except AttributeError:
	84	sys.stderr.write("bsddb module unavailable.\n")
	85	return 1
	86	elif opt in ("-b", "--btree"):
	87	try:
	88	dbopen = bsddb.btopen
	89	except AttributeError:
	90	sys.stderr.write("bsddb module unavailable.\n")
	91	return 1
	92	elif opt in ("-r", "--recno"):
	93	try:
	94	dbopen = bsddb.rnopen
	95	except AttributeError:
	96	sys.stderr.write("bsddb module unavailable.\n")
	97	return 1
	98	elif opt in ("-a", "--anydbm"):
	99	try:
	100	dbopen = anydbm.open
	101	except AttributeError:
	102	sys.stderr.write("anydbm module unavailable.\n")
	103	return 1
	104	elif opt in ("-g", "--gdbm"):
	105	try:
	106	dbopen = gdbm.open
	107	except AttributeError:
	108	sys.stderr.write("gdbm module unavailable.\n")
	109	return 1
	110	elif opt in ("-d", "--dbm"):
	111	try:
	112	dbopen = dbm.open
	113	except AttributeError:
	114	sys.stderr.write("dbm module unavailable.\n")
	115	return 1
	116	if dbopen is None:
	117	if bsddb is None:
	118	sys.stderr.write("bsddb module unavailable - ")
	119	sys.stderr.write("must specify dbtype.\n")
	120	return 1
	121	else:
	122	dbopen = bsddb.hashopen
	123
	124	try:
	125	db = dbopen(dbfile, 'c')
	126	except bsddb.error:
	127	sys.stderr.write("Unable to open %s. " % dbfile)
	128	sys.stderr.write("Check for format or version mismatch.\n")
	129	return 1
	130	else:
	131	for k in db.keys():
	132	del db[k]
	133
	134	while 1:
	135	try:
	136	(key, val) = pickle.load(pfile)
	137	except EOFError:
	138	break
	139	db[key] = val
	140
	141	db.close()
	142	pfile.close()
	143
	144	return 0
	145
	146	if __name__ == "__main__":
	147	sys.exit(main(sys.argv[1:]))

sys/src/cmd/python/Tools/scripts/pindent.py+543−0

@@ -0,0 +1,543 @@
	1	#! /usr/bin/env python
	2
	3	# This file contains a class and a main program that perform three
	4	# related (though complimentary) formatting operations on Python
	5	# programs. When called as "pindent -c", it takes a valid Python
	6	# program as input and outputs a version augmented with block-closing
	7	# comments. When called as "pindent -d", it assumes its input is a
	8	# Python program with block-closing comments and outputs a commentless
	9	# version. When called as "pindent -r" it assumes its input is a
	10	# Python program with block-closing comments but with its indentation
	11	# messed up, and outputs a properly indented version.
	12
	13	# A "block-closing comment" is a comment of the form '# end <keyword>'
	14	# where <keyword> is the keyword that opened the block. If the
	15	# opening keyword is 'def' or 'class', the function or class name may
	16	# be repeated in the block-closing comment as well. Here is an
	17	# example of a program fully augmented with block-closing comments:
	18
	19	# def foobar(a, b):
	20	# if a == b:
	21	# a = a+1
	22	# elif a < b:
	23	# b = b-1
	24	# if b > a: a = a-1
	25	# # end if
	26	# else:
	27	# print 'oops!'
	28	# # end if
	29	# # end def foobar
	30
	31	# Note that only the last part of an if...elif...else... block needs a
	32	# block-closing comment; the same is true for other compound
	33	# statements (e.g. try...except). Also note that "short-form" blocks
	34	# like the second 'if' in the example must be closed as well;
	35	# otherwise the 'else' in the example would be ambiguous (remember
	36	# that indentation is not significant when interpreting block-closing
	37	# comments).
	38
	39	# The operations are idempotent (i.e. applied to their own output
	40	# they yield an identical result). Running first "pindent -c" and
	41	# then "pindent -r" on a valid Python program produces a program that
	42	# is semantically identical to the input (though its indentation may
	43	# be different). Running "pindent -e" on that output produces a
	44	# program that only differs from the original in indentation.
	45
	46	# Other options:
	47	# -s stepsize: set the indentation step size (default 8)
	48	# -t tabsize : set the number of spaces a tab character is worth (default 8)
	49	# -e : expand TABs into spaces
	50	# file ... : input file(s) (default standard input)
	51	# The results always go to standard output
	52
	53	# Caveats:
	54	# - comments ending in a backslash will be mistaken for continued lines
	55	# - continuations using backslash are always left unchanged
	56	# - continuations inside parentheses are not extra indented by -r
	57	# but must be indented for -c to work correctly (this breaks
	58	# idempotency!)
	59	# - continued lines inside triple-quoted strings are totally garbled
	60
	61	# Secret feature:
	62	# - On input, a block may also be closed with an "end statement" --
	63	# this is a block-closing comment without the '#' sign.
	64
	65	# Possible improvements:
	66	# - check syntax based on transitions in 'next' table
	67	# - better error reporting
	68	# - better error recovery
	69	# - check identifier after class/def
	70
	71	# The following wishes need a more complete tokenization of the source:
	72	# - Don't get fooled by comments ending in backslash
	73	# - reindent continuation lines indicated by backslash
	74	# - handle continuation lines inside parentheses/braces/brackets
	75	# - handle triple quoted strings spanning lines
	76	# - realign comments
	77	# - optionally do much more thorough reformatting, a la C indent
	78
	79	# Defaults
	80	STEPSIZE = 8
	81	TABSIZE = 8
	82	EXPANDTABS = 0
	83
	84	import os
	85	import re
	86	import sys
	87
	88	next = {}
	89	next['if'] = next['elif'] = 'elif', 'else', 'end'
	90	next['while'] = next['for'] = 'else', 'end'
	91	next['try'] = 'except', 'finally'
	92	next['except'] = 'except', 'else', 'end'
	93	next['else'] = next['finally'] = next['def'] = next['class'] = 'end'
	94	next['end'] = ()
	95	start = 'if', 'while', 'for', 'try', 'def', 'class'
	96
	97	class PythonIndenter:
	98
	99	def __init__(self, fpi = sys.stdin, fpo = sys.stdout,
	100	indentsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
	101	self.fpi = fpi
	102	self.fpo = fpo
	103	self.indentsize = indentsize
	104	self.tabsize = tabsize
	105	self.lineno = 0
	106	self.expandtabs = expandtabs
	107	self._write = fpo.write
	108	self.kwprog = re.compile(
	109	r'^\s*(?P<kw>[a-z]+)'
	110	r'(\s+(?P<id>[a-zA-Z_]\w*))?'
	111	r'[^\w]')
	112	self.endprog = re.compile(
	113	r'^\s#?\send\s+(?P<kw>[a-z]+)'
	114	r'(\s+(?P<id>[a-zA-Z_]\w*))?'
	115	r'[^\w]')
	116	self.wsprog = re.compile(r'^[ \t]*')
	117	# end def __init__
	118
	119	def write(self, line):
	120	if self.expandtabs:
	121	self._write(line.expandtabs(self.tabsize))
	122	else:
	123	self._write(line)
	124	# end if
	125	# end def write
	126
	127	def readline(self):
	128	line = self.fpi.readline()
	129	if line: self.lineno = self.lineno + 1
	130	# end if
	131	return line
	132	# end def readline
	133
	134	def error(self, fmt, *args):
	135	if args: fmt = fmt % args
	136	# end if
	137	sys.stderr.write('Error at line %d: %s\n' % (self.lineno, fmt))
	138	self.write('### %s ###\n' % fmt)
	139	# end def error
	140
	141	def getline(self):
	142	line = self.readline()
	143	while line[-2:] == '\\\n':
	144	line2 = self.readline()
	145	if not line2: break
	146	# end if
	147	line = line + line2
	148	# end while
	149	return line
	150	# end def getline
	151
	152	def putline(self, line, indent = None):
	153	if indent is None:
	154	self.write(line)
	155	return
	156	# end if
	157	tabs, spaces = divmod(indent*self.indentsize, self.tabsize)
	158	i = 0
	159	m = self.wsprog.match(line)
	160	if m: i = m.end()
	161	# end if
	162	self.write('\t'tabs + ' 'spaces + line[i:])
	163	# end def putline
	164
	165	def reformat(self):
	166	stack = []
	167	while 1:
	168	line = self.getline()
	169	if not line: break # EOF
	170	# end if
	171	m = self.endprog.match(line)
	172	if m:
	173	kw = 'end'
	174	kw2 = m.group('kw')
	175	if not stack:
	176	self.error('unexpected end')
	177	elif stack[-1][0] != kw2:
	178	self.error('unmatched end')
	179	# end if
	180	del stack[-1:]
	181	self.putline(line, len(stack))
	182	continue
	183	# end if
	184	m = self.kwprog.match(line)
	185	if m:
	186	kw = m.group('kw')
	187	if kw in start:
	188	self.putline(line, len(stack))
	189	stack.append((kw, kw))
	190	continue
	191	# end if
	192	if next.has_key(kw) and stack:
	193	self.putline(line, len(stack)-1)
	194	kwa, kwb = stack[-1]
	195	stack[-1] = kwa, kw
	196	continue
	197	# end if
	198	# end if
	199	self.putline(line, len(stack))
	200	# end while
	201	if stack:
	202	self.error('unterminated keywords')
	203	for kwa, kwb in stack:
	204	self.write('\t%s\n' % kwa)
	205	# end for
	206	# end if
	207	# end def reformat
	208
	209	def delete(self):
	210	begin_counter = 0
	211	end_counter = 0
	212	while 1:
	213	line = self.getline()
	214	if not line: break # EOF
	215	# end if
	216	m = self.endprog.match(line)
	217	if m:
	218	end_counter = end_counter + 1
	219	continue
	220	# end if
	221	m = self.kwprog.match(line)
	222	if m:
	223	kw = m.group('kw')
	224	if kw in start:
	225	begin_counter = begin_counter + 1
	226	# end if
	227	# end if
	228	self.putline(line)
	229	# end while
	230	if begin_counter - end_counter < 0:
	231	sys.stderr.write('Warning: input contained more end tags than expected\n')
	232	elif begin_counter - end_counter > 0:
	233	sys.stderr.write('Warning: input contained less end tags than expected\n')
	234	# end if
	235	# end def delete
	236
	237	def complete(self):
	238	self.indentsize = 1
	239	stack = []
	240	todo = []
	241	thisid = ''
	242	current, firstkw, lastkw, topid = 0, '', '', ''
	243	while 1:
	244	line = self.getline()
	245	i = 0
	246	m = self.wsprog.match(line)
	247	if m: i = m.end()
	248	# end if
	249	m = self.endprog.match(line)
	250	if m:
	251	thiskw = 'end'
	252	endkw = m.group('kw')
	253	thisid = m.group('id')
	254	else:
	255	m = self.kwprog.match(line)
	256	if m:
	257	thiskw = m.group('kw')
	258	if not next.has_key(thiskw):
	259	thiskw = ''
	260	# end if
	261	if thiskw in ('def', 'class'):
	262	thisid = m.group('id')
	263	else:
	264	thisid = ''
	265	# end if
	266	elif line[i:i+1] in ('\n', '#'):
	267	todo.append(line)
	268	continue
	269	else:
	270	thiskw = ''
	271	# end if
	272	# end if
	273	indent = len(line[:i].expandtabs(self.tabsize))
	274	while indent < current:
	275	if firstkw:
	276	if topid:
	277	s = '# end %s %s\n' % (
	278	firstkw, topid)
	279	else:
	280	s = '# end %s\n' % firstkw
	281	# end if
	282	self.putline(s, current)
	283	firstkw = lastkw = ''
	284	# end if
	285	current, firstkw, lastkw, topid = stack[-1]
	286	del stack[-1]
	287	# end while
	288	if indent == current and firstkw:
	289	if thiskw == 'end':
	290	if endkw != firstkw:
	291	self.error('mismatched end')
	292	# end if
	293	firstkw = lastkw = ''
	294	elif not thiskw or thiskw in start:
	295	if topid:
	296	s = '# end %s %s\n' % (
	297	firstkw, topid)
	298	else:
	299	s = '# end %s\n' % firstkw
	300	# end if
	301	self.putline(s, current)
	302	firstkw = lastkw = topid = ''
	303	# end if
	304	# end if
	305	if indent > current:
	306	stack.append((current, firstkw, lastkw, topid))
	307	if thiskw and thiskw not in start:
	308	# error
	309	thiskw = ''
	310	# end if
	311	current, firstkw, lastkw, topid = \
	312	indent, thiskw, thiskw, thisid
	313	# end if
	314	if thiskw:
	315	if thiskw in start:
	316	firstkw = lastkw = thiskw
	317	topid = thisid
	318	else:
	319	lastkw = thiskw
	320	# end if
	321	# end if
	322	for l in todo: self.write(l)
	323	# end for
	324	todo = []
	325	if not line: break
	326	# end if
	327	self.write(line)
	328	# end while
	329	# end def complete
	330
	331	# end class PythonIndenter
	332
	333	# Simplified user interface
	334	# - xxx_filter(input, output): read and write file objects
	335	# - xxx_string(s): take and return string object
	336	# - xxx_file(filename): process file in place, return true iff changed
	337
	338	def complete_filter(input = sys.stdin, output = sys.stdout,
	339	stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
	340	pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
	341	pi.complete()
	342	# end def complete_filter
	343
	344	def delete_filter(input= sys.stdin, output = sys.stdout,
	345	stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
	346	pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
	347	pi.delete()
	348	# end def delete_filter
	349
	350	def reformat_filter(input = sys.stdin, output = sys.stdout,
	351	stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
	352	pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
	353	pi.reformat()
	354	# end def reformat_filter
	355
	356	class StringReader:
	357	def __init__(self, buf):
	358	self.buf = buf
	359	self.pos = 0
	360	self.len = len(self.buf)
	361	# end def __init__
	362	def read(self, n = 0):
	363	if n <= 0:
	364	n = self.len - self.pos
	365	else:
	366	n = min(n, self.len - self.pos)
	367	# end if
	368	r = self.buf[self.pos : self.pos + n]
	369	self.pos = self.pos + n
	370	return r
	371	# end def read
	372	def readline(self):
	373	i = self.buf.find('\n', self.pos)
	374	return self.read(i + 1 - self.pos)
	375	# end def readline
	376	def readlines(self):
	377	lines = []
	378	line = self.readline()
	379	while line:
	380	lines.append(line)
	381	line = self.readline()
	382	# end while
	383	return lines
	384	# end def readlines
	385	# seek/tell etc. are left as an exercise for the reader
	386	# end class StringReader
	387
	388	class StringWriter:
	389	def __init__(self):
	390	self.buf = ''
	391	# end def __init__
	392	def write(self, s):
	393	self.buf = self.buf + s
	394	# end def write
	395	def getvalue(self):
	396	return self.buf
	397	# end def getvalue
	398	# end class StringWriter
	399
	400	def complete_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
	401	input = StringReader(source)
	402	output = StringWriter()
	403	pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
	404	pi.complete()
	405	return output.getvalue()
	406	# end def complete_string
	407
	408	def delete_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
	409	input = StringReader(source)
	410	output = StringWriter()
	411	pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
	412	pi.delete()
	413	return output.getvalue()
	414	# end def delete_string
	415
	416	def reformat_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
	417	input = StringReader(source)
	418	output = StringWriter()
	419	pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
	420	pi.reformat()
	421	return output.getvalue()
	422	# end def reformat_string
	423
	424	def complete_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
	425	source = open(filename, 'r').read()
	426	result = complete_string(source, stepsize, tabsize, expandtabs)
	427	if source == result: return 0
	428	# end if
	429	import os
	430	try: os.rename(filename, filename + '~')
	431	except os.error: pass
	432	# end try
	433	f = open(filename, 'w')
	434	f.write(result)
	435	f.close()
	436	return 1
	437	# end def complete_file
	438
	439	def delete_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
	440	source = open(filename, 'r').read()
	441	result = delete_string(source, stepsize, tabsize, expandtabs)
	442	if source == result: return 0
	443	# end if
	444	import os
	445	try: os.rename(filename, filename + '~')
	446	except os.error: pass
	447	# end try
	448	f = open(filename, 'w')
	449	f.write(result)
	450	f.close()
	451	return 1
	452	# end def delete_file
	453
	454	def reformat_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
	455	source = open(filename, 'r').read()
	456	result = reformat_string(source, stepsize, tabsize, expandtabs)
	457	if source == result: return 0
	458	# end if
	459	import os
	460	try: os.rename(filename, filename + '~')
	461	except os.error: pass
	462	# end try
	463	f = open(filename, 'w')
	464	f.write(result)
	465	f.close()
	466	return 1
	467	# end def reformat_file
	468
	469	# Test program when called as a script
	470
	471	usage = """
	472	usage: pindent (-c\|-d\|-r) [-s stepsize] [-t tabsize] [-e] [file] ...
	473	-c : complete a correctly indented program (add #end directives)
	474	-d : delete #end directives
	475	-r : reformat a completed program (use #end directives)
	476	-s stepsize: indentation step (default %(STEPSIZE)d)
	477	-t tabsize : the worth in spaces of a tab (default %(TABSIZE)d)
	478	-e : expand TABs into spaces (defailt OFF)
	479	[file] ... : files are changed in place, with backups in file~
	480	If no files are specified or a single - is given,
	481	the program acts as a filter (reads stdin, writes stdout).
	482	""" % vars()
	483
	484	def error_both(op1, op2):
	485	sys.stderr.write('Error: You can not specify both '+op1+' and -'+op2[0]+' at the same time\n')
	486	sys.stderr.write(usage)
	487	sys.exit(2)
	488	# end def error_both
	489
	490	def test():
	491	import getopt
	492	try:
	493	opts, args = getopt.getopt(sys.argv[1:], 'cdrs:t:e')
	494	except getopt.error, msg:
	495	sys.stderr.write('Error: %s\n' % msg)
	496	sys.stderr.write(usage)
	497	sys.exit(2)
	498	# end try
	499	action = None
	500	stepsize = STEPSIZE
	501	tabsize = TABSIZE
	502	expandtabs = EXPANDTABS
	503	for o, a in opts:
	504	if o == '-c':
	505	if action: error_both(o, action)
	506	# end if
	507	action = 'complete'
	508	elif o == '-d':
	509	if action: error_both(o, action)
	510	# end if
	511	action = 'delete'
	512	elif o == '-r':
	513	if action: error_both(o, action)
	514	# end if
	515	action = 'reformat'
	516	elif o == '-s':
	517	stepsize = int(a)
	518	elif o == '-t':
	519	tabsize = int(a)
	520	elif o == '-e':
	521	expandtabs = 1
	522	# end if
	523	# end for
	524	if not action:
	525	sys.stderr.write(
	526	'You must specify -c(omplete), -d(elete) or -r(eformat)\n')
	527	sys.stderr.write(usage)
	528	sys.exit(2)
	529	# end if
	530	if not args or args == ['-']:
	531	action = eval(action + '_filter')
	532	action(sys.stdin, sys.stdout, stepsize, tabsize, expandtabs)
	533	else:
	534	action = eval(action + '_file')
	535	for filename in args:
	536	action(filename, stepsize, tabsize, expandtabs)
	537	# end for
	538	# end if
	539	# end def test
	540
	541	if __name__ == '__main__':
	542	test()
	543	# end if

sys/src/cmd/python/Tools/scripts/ptags.py+53−0

@@ -0,0 +1,53 @@
	1	#! /usr/bin/env python
	2
	3	# ptags
	4	#
	5	# Create a tags file for Python programs, usable with vi.
	6	# Tagged are:
	7	# - functions (even inside other defs or classes)
	8	# - classes
	9	# - filenames
	10	# Warns about files it cannot open.
	11	# No warnings about duplicate tags.
	12
	13	import sys, re, os
	14
	15	tags = [] # Modified global variable!
	16
	17	def main():
	18	args = sys.argv[1:]
	19	for filename in args:
	20	treat_file(filename)
	21	if tags:
	22	fp = open('tags', 'w')
	23	tags.sort()
	24	for s in tags: fp.write(s)
	25
	26
	27	expr = '^[ \t](def\|class)[ \t]+([a-zA-Z0-9_]+)[ \t][:\(]'
	28	matcher = re.compile(expr)
	29
	30	def treat_file(filename):
	31	try:
	32	fp = open(filename, 'r')
	33	except:
	34	sys.stderr.write('Cannot open %s\n' % filename)
	35	return
	36	base = os.path.basename(filename)
	37	if base[-3:] == '.py':
	38	base = base[:-3]
	39	s = base + '\t' + filename + '\t' + '1\n'
	40	tags.append(s)
	41	while 1:
	42	line = fp.readline()
	43	if not line:
	44	break
	45	m = matcher.match(line)
	46	if m:
	47	content = m.group(0)
	48	name = m.group(2)
	49	s = name + '\t' + filename + '\t/^' + content + '/\n'
	50	tags.append(s)
	51
	52	if __name__ == '__main__':
	53	main()

sys/src/cmd/python/Tools/scripts/pydoc+5−0

@@ -0,0 +1,5 @@
	1	#!/usr/bin/env python
	2
	3	import pydoc
	4	if __name__ == '__main__':
	5	pydoc.cli()

sys/src/cmd/python/Tools/scripts/pydocgui.pyw+7−0

@@ -0,0 +1,7 @@
	1	# Note: this file must not be named pydoc.pyw, lest it just end up
	2	# importing itself (Python began allowing import of .pyw files
	3	# between 2.2a1 and 2.2a2).
	4	import pydoc
	5
	6	if __name__ == '__main__':
	7	pydoc.gui()

sys/src/cmd/python/Tools/scripts/pysource.py+130−0

@@ -0,0 +1,130 @@
	1	#!/usr/bin/env python
	2
	3	"""\
	4	List python source files.
	5
	6	There are three functions to check whether a file is a Python source, listed
	7	here with increasing complexity:
	8
	9	- has_python_ext() checks whether a file name ends in '.py[w]'.
	10	- look_like_python() checks whether the file is not binary and either has
	11	the '.py[w]' extension or the first line contains the word 'python'.
	12	- can_be_compiled() checks whether the file can be compiled by compile().
	13
	14	The file also must be of appropriate size - not bigger than a megabyte.
	15
	16	walk_python_files() recursively lists all Python files under the given directories.
	17	"""
	18	__author__ = "Oleg Broytmann, Georg Brandl"
	19
	20	__all__ = ["has_python_ext", "looks_like_python", "can_be_compiled", "walk_python_files"]
	21
	22
	23	import sys, os, re
	24
	25	binary_re = re.compile('[\x00-\x08\x0E-\x1F\x7F]')
	26
	27	debug = False
	28
	29	def print_debug(msg):
	30	if debug: print msg
	31
	32
	33	def _open(fullpath):
	34	try:
	35	size = os.stat(fullpath).st_size
	36	except OSError, err: # Permission denied - ignore the file
	37	print_debug("%s: permission denied: %s" % (fullpath, err))
	38	return None
	39
	40	if size > 1024*1024: # too big
	41	print_debug("%s: the file is too big: %d bytes" % (fullpath, size))
	42	return None
	43
	44	try:
	45	return open(fullpath, 'rU')
	46	except IOError, err: # Access denied, or a special file - ignore it
	47	print_debug("%s: access denied: %s" % (fullpath, err))
	48	return None
	49
	50	def has_python_ext(fullpath):
	51	return fullpath.endswith(".py") or fullpath.endswith(".pyw")
	52
	53	def looks_like_python(fullpath):
	54	infile = _open(fullpath)
	55	if infile is None:
	56	return False
	57
	58	line = infile.readline()
	59	infile.close()
	60
	61	if binary_re.search(line):
	62	# file appears to be binary
	63	print_debug("%s: appears to be binary" % fullpath)
	64	return False
	65
	66	if fullpath.endswith(".py") or fullpath.endswith(".pyw"):
	67	return True
	68	elif "python" in line:
	69	# disguised Python script (e.g. CGI)
	70	return True
	71
	72	return False
	73
	74	def can_be_compiled(fullpath):
	75	infile = _open(fullpath)
	76	if infile is None:
	77	return False
	78
	79	code = infile.read()
	80	infile.close()
	81
	82	try:
	83	compile(code, fullpath, "exec")
	84	except Exception, err:
	85	print_debug("%s: cannot compile: %s" % (fullpath, err))
	86	return False
	87
	88	return True
	89
	90
	91	def walk_python_files(paths, is_python=looks_like_python, exclude_dirs=None):
	92	"""\
	93	Recursively yield all Python source files below the given paths.
	94
	95	paths: a list of files and/or directories to be checked.
	96	is_python: a function that takes a file name and checks whether it is a
	97	Python source file
	98	exclude_dirs: a list of directory base names that should be excluded in
	99	the search
	100	"""
	101	if exclude_dirs is None:
	102	exclude_dirs=[]
	103
	104	for path in paths:
	105	print_debug("testing: %s" % path)
	106	if os.path.isfile(path):
	107	if is_python(path):
	108	yield path
	109	elif os.path.isdir(path):
	110	print_debug(" it is a directory")
	111	for dirpath, dirnames, filenames in os.walk(path):
	112	for exclude in exclude_dirs:
	113	if exclude in dirnames:
	114	dirnames.remove(exclude)
	115	for filename in filenames:
	116	fullpath = os.path.join(dirpath, filename)
	117	print_debug("testing: %s" % fullpath)
	118	if is_python(fullpath):
	119	yield fullpath
	120	else:
	121	print_debug(" unknown type")
	122
	123
	124	if __name__ == "__main__":
	125	# Two simple examples/tests
	126	for fullpath in walk_python_files(['.']):
	127	print fullpath
	128	print "----------"
	129	for fullpath in walk_python_files(['.'], is_python=can_be_compiled):
	130	print fullpath

sys/src/cmd/python/Tools/scripts/redemo.py+171−0

@@ -0,0 +1,171 @@
	1	"""Basic regular expression demostration facility (Perl style syntax)."""
	2
	3	from Tkinter import *
	4	import re
	5
	6	class ReDemo:
	7
	8	def __init__(self, master):
	9	self.master = master
	10
	11	self.promptdisplay = Label(self.master, anchor=W,
	12	text="Enter a Perl-style regular expression:")
	13	self.promptdisplay.pack(side=TOP, fill=X)
	14
	15	self.regexdisplay = Entry(self.master)
	16	self.regexdisplay.pack(fill=X)
	17	self.regexdisplay.focus_set()
	18
	19	self.addoptions()
	20
	21	self.statusdisplay = Label(self.master, text="", anchor=W)
	22	self.statusdisplay.pack(side=TOP, fill=X)
	23
	24	self.labeldisplay = Label(self.master, anchor=W,
	25	text="Enter a string to search:")
	26	self.labeldisplay.pack(fill=X)
	27	self.labeldisplay.pack(fill=X)
	28
	29	self.showframe = Frame(master)
	30	self.showframe.pack(fill=X, anchor=W)
	31
	32	self.showvar = StringVar(master)
	33	self.showvar.set("first")
	34
	35	self.showfirstradio = Radiobutton(self.showframe,
	36	text="Highlight first match",
	37	variable=self.showvar,
	38	value="first",
	39	command=self.recompile)
	40	self.showfirstradio.pack(side=LEFT)
	41
	42	self.showallradio = Radiobutton(self.showframe,
	43	text="Highlight all matches",
	44	variable=self.showvar,
	45	value="all",
	46	command=self.recompile)
	47	self.showallradio.pack(side=LEFT)
	48
	49	self.stringdisplay = Text(self.master, width=60, height=4)
	50	self.stringdisplay.pack(fill=BOTH, expand=1)
	51	self.stringdisplay.tag_configure("hit", background="yellow")
	52
	53	self.grouplabel = Label(self.master, text="Groups:", anchor=W)
	54	self.grouplabel.pack(fill=X)
	55
	56	self.grouplist = Listbox(self.master)
	57	self.grouplist.pack(expand=1, fill=BOTH)
	58
	59	self.regexdisplay.bind('<Key>', self.recompile)
	60	self.stringdisplay.bind('<Key>', self.reevaluate)
	61
	62	self.compiled = None
	63	self.recompile()
	64
	65	btags = self.regexdisplay.bindtags()
	66	self.regexdisplay.bindtags(btags[1:] + btags[:1])
	67
	68	btags = self.stringdisplay.bindtags()
	69	self.stringdisplay.bindtags(btags[1:] + btags[:1])
	70
	71	def addoptions(self):
	72	self.frames = []
	73	self.boxes = []
	74	self.vars = []
	75	for name in ('IGNORECASE',
	76	'LOCALE',
	77	'MULTILINE',
	78	'DOTALL',
	79	'VERBOSE'):
	80	if len(self.boxes) % 3 == 0:
	81	frame = Frame(self.master)
	82	frame.pack(fill=X)
	83	self.frames.append(frame)
	84	val = getattr(re, name)
	85	var = IntVar()
	86	box = Checkbutton(frame,
	87	variable=var, text=name,
	88	offvalue=0, onvalue=val,
	89	command=self.recompile)
	90	box.pack(side=LEFT)
	91	self.boxes.append(box)
	92	self.vars.append(var)
	93
	94	def getflags(self):
	95	flags = 0
	96	for var in self.vars:
	97	flags = flags \| var.get()
	98	flags = flags
	99	return flags
	100
	101	def recompile(self, event=None):
	102	try:
	103	self.compiled = re.compile(self.regexdisplay.get(),
	104	self.getflags())
	105	bg = self.promptdisplay['background']
	106	self.statusdisplay.config(text="", background=bg)
	107	except re.error, msg:
	108	self.compiled = None
	109	self.statusdisplay.config(
	110	text="re.error: %s" % str(msg),
	111	background="red")
	112	self.reevaluate()
	113
	114	def reevaluate(self, event=None):
	115	try:
	116	self.stringdisplay.tag_remove("hit", "1.0", END)
	117	except TclError:
	118	pass
	119	try:
	120	self.stringdisplay.tag_remove("hit0", "1.0", END)
	121	except TclError:
	122	pass
	123	self.grouplist.delete(0, END)
	124	if not self.compiled:
	125	return
	126	self.stringdisplay.tag_configure("hit", background="yellow")
	127	self.stringdisplay.tag_configure("hit0", background="orange")
	128	text = self.stringdisplay.get("1.0", END)
	129	last = 0
	130	nmatches = 0
	131	while last <= len(text):
	132	m = self.compiled.search(text, last)
	133	if m is None:
	134	break
	135	first, last = m.span()
	136	if last == first:
	137	last = first+1
	138	tag = "hit0"
	139	else:
	140	tag = "hit"
	141	pfirst = "1.0 + %d chars" % first
	142	plast = "1.0 + %d chars" % last
	143	self.stringdisplay.tag_add(tag, pfirst, plast)
	144	if nmatches == 0:
	145	self.stringdisplay.yview_pickplace(pfirst)
	146	groups = list(m.groups())
	147	groups.insert(0, m.group())
	148	for i in range(len(groups)):
	149	g = "%2d: %r" % (i, groups[i])
	150	self.grouplist.insert(END, g)
	151	nmatches = nmatches + 1
	152	if self.showvar.get() == "first":
	153	break
	154
	155	if nmatches == 0:
	156	self.statusdisplay.config(text="(no match)",
	157	background="yellow")
	158	else:
	159	self.statusdisplay.config(text="")
	160
	161
	162	# Main function, run when invoked as a stand-alone Python program.
	163
	164	def main():
	165	root = Tk()
	166	demo = ReDemo(root)
	167	root.protocol('WM_DELETE_WINDOW', root.quit)
	168	root.mainloop()
	169
	170	if __name__ == '__main__':
	171	main()

sys/src/cmd/python/Tools/scripts/reindent.py+293−0

@@ -0,0 +1,293 @@
	1	#! /usr/bin/env python
	2
	3	# Released to the public domain, by Tim Peters, 03 October 2000.
	4
	5	"""reindent [-d][-r][-v] [ path ... ]
	6
	7	-d (--dryrun) Dry run. Analyze, but don't make any changes to, files.
	8	-r (--recurse) Recurse. Search for all .py files in subdirectories too.
	9	-v (--verbose) Verbose. Print informative msgs; else no output.
	10	-h (--help) Help. Print this usage information and exit.
	11
	12	Change Python (.py) files to use 4-space indents and no hard tab characters.
	13	Also trim excess spaces and tabs from ends of lines, and remove empty lines
	14	at the end of files. Also ensure the last line ends with a newline.
	15
	16	If no paths are given on the command line, reindent operates as a filter,
	17	reading a single source file from standard input and writing the transformed
	18	source to standard output. In this case, the -d, -r and -v flags are
	19	ignored.
	20
	21	You can pass one or more file and/or directory paths. When a directory
	22	path, all .py files within the directory will be examined, and, if the -r
	23	option is given, likewise recursively for subdirectories.
	24
	25	If output is not to standard output, reindent overwrites files in place,
	26	renaming the originals with a .bak extension. If it finds nothing to
	27	change, the file is left alone. If reindent does change a file, the changed
	28	file is a fixed-point for future runs (i.e., running reindent on the
	29	resulting .py file won't change it again).
	30
	31	The hard part of reindenting is figuring out what to do with comment
	32	lines. So long as the input files get a clean bill of health from
	33	tabnanny.py, reindent should do a good job.
	34	"""
	35
	36	__version__ = "1"
	37
	38	import tokenize
	39	import os
	40	import sys
	41
	42	verbose = 0
	43	recurse = 0
	44	dryrun = 0
	45
	46	def usage(msg=None):
	47	if msg is not None:
	48	print >> sys.stderr, msg
	49	print >> sys.stderr, __doc__
	50
	51	def errprint(*args):
	52	sep = ""
	53	for arg in args:
	54	sys.stderr.write(sep + str(arg))
	55	sep = " "
	56	sys.stderr.write("\n")
	57
	58	def main():
	59	import getopt
	60	global verbose, recurse, dryrun
	61	try:
	62	opts, args = getopt.getopt(sys.argv[1:], "drvh",
	63	["dryrun", "recurse", "verbose", "help"])
	64	except getopt.error, msg:
	65	usage(msg)
	66	return
	67	for o, a in opts:
	68	if o in ('-d', '--dryrun'):
	69	dryrun += 1
	70	elif o in ('-r', '--recurse'):
	71	recurse += 1
	72	elif o in ('-v', '--verbose'):
	73	verbose += 1
	74	elif o in ('-h', '--help'):
	75	usage()
	76	return
	77	if not args:
	78	r = Reindenter(sys.stdin)
	79	r.run()
	80	r.write(sys.stdout)
	81	return
	82	for arg in args:
	83	check(arg)
	84
	85	def check(file):
	86	if os.path.isdir(file) and not os.path.islink(file):
	87	if verbose:
	88	print "listing directory", file
	89	names = os.listdir(file)
	90	for name in names:
	91	fullname = os.path.join(file, name)
	92	if ((recurse and os.path.isdir(fullname) and
	93	not os.path.islink(fullname))
	94	or name.lower().endswith(".py")):
	95	check(fullname)
	96	return
	97
	98	if verbose:
	99	print "checking", file, "...",
	100	try:
	101	f = open(file)
	102	except IOError, msg:
	103	errprint("%s: I/O Error: %s" % (file, str(msg)))
	104	return
	105
	106	r = Reindenter(f)
	107	f.close()
	108	if r.run():
	109	if verbose:
	110	print "changed."
	111	if dryrun:
	112	print "But this is a dry run, so leaving it alone."
	113	if not dryrun:
	114	bak = file + ".bak"
	115	if os.path.exists(bak):
	116	os.remove(bak)
	117	os.rename(file, bak)
	118	if verbose:
	119	print "renamed", file, "to", bak
	120	f = open(file, "w")
	121	r.write(f)
	122	f.close()
	123	if verbose:
	124	print "wrote new", file
	125	else:
	126	if verbose:
	127	print "unchanged."
	128
	129	def _rstrip(line, JUNK='\n \t'):
	130	"""Return line stripped of trailing spaces, tabs, newlines.
	131
	132	Note that line.rstrip() instead also strips sundry control characters,
	133	but at least one known Emacs user expects to keep junk like that, not
	134	mentioning Barry by name or anything <wink>.
	135	"""
	136
	137	i = len(line)
	138	while i > 0 and line[i-1] in JUNK:
	139	i -= 1
	140	return line[:i]
	141
	142	class Reindenter:
	143
	144	def __init__(self, f):
	145	self.find_stmt = 1 # next token begins a fresh stmt?
	146	self.level = 0 # current indent level
	147
	148	# Raw file lines.
	149	self.raw = f.readlines()
	150
	151	# File lines, rstripped & tab-expanded. Dummy at start is so
	152	# that we can use tokenize's 1-based line numbering easily.
	153	# Note that a line is all-blank iff it's "\n".
	154	self.lines = [_rstrip(line).expandtabs() + "\n"
	155	for line in self.raw]
	156	self.lines.insert(0, None)
	157	self.index = 1 # index into self.lines of next line
	158
	159	# List of (lineno, indentlevel) pairs, one for each stmt and
	160	# comment line. indentlevel is -1 for comment lines, as a
	161	# signal that tokenize doesn't know what to do about them;
	162	# indeed, they're our headache!
	163	self.stats = []
	164
	165	def run(self):
	166	tokenize.tokenize(self.getline, self.tokeneater)
	167	# Remove trailing empty lines.
	168	lines = self.lines
	169	while lines and lines[-1] == "\n":
	170	lines.pop()
	171	# Sentinel.
	172	stats = self.stats
	173	stats.append((len(lines), 0))
	174	# Map count of leading spaces to # we want.
	175	have2want = {}
	176	# Program after transformation.
	177	after = self.after = []
	178	# Copy over initial empty lines -- there's nothing to do until
	179	# we see a line with something on it.
	180	i = stats[0][0]
	181	after.extend(lines[1:i])
	182	for i in range(len(stats)-1):
	183	thisstmt, thislevel = stats[i]
	184	nextstmt = stats[i+1][0]
	185	have = getlspace(lines[thisstmt])
	186	want = thislevel * 4
	187	if want < 0:
	188	# A comment line.
	189	if have:
	190	# An indented comment line. If we saw the same
	191	# indentation before, reuse what it most recently
	192	# mapped to.
	193	want = have2want.get(have, -1)
	194	if want < 0:
	195	# Then it probably belongs to the next real stmt.
	196	for j in xrange(i+1, len(stats)-1):
	197	jline, jlevel = stats[j]
	198	if jlevel >= 0:
	199	if have == getlspace(lines[jline]):
	200	want = jlevel * 4
	201	break
	202	if want < 0: # Maybe it's a hanging
	203	# comment like this one,
	204	# in which case we should shift it like its base
	205	# line got shifted.
	206	for j in xrange(i-1, -1, -1):
	207	jline, jlevel = stats[j]
	208	if jlevel >= 0:
	209	want = have + getlspace(after[jline-1]) - \
	210	getlspace(lines[jline])
	211	break
	212	if want < 0:
	213	# Still no luck -- leave it alone.
	214	want = have
	215	else:
	216	want = 0
	217	assert want >= 0
	218	have2want[have] = want
	219	diff = want - have
	220	if diff == 0 or have == 0:
	221	after.extend(lines[thisstmt:nextstmt])
	222	else:
	223	for line in lines[thisstmt:nextstmt]:
	224	if diff > 0:
	225	if line == "\n":
	226	after.append(line)
	227	else:
	228	after.append(" " * diff + line)
	229	else:
	230	remove = min(getlspace(line), -diff)
	231	after.append(line[remove:])
	232	return self.raw != self.after
	233
	234	def write(self, f):
	235	f.writelines(self.after)
	236
	237	# Line-getter for tokenize.
	238	def getline(self):
	239	if self.index >= len(self.lines):
	240	line = ""
	241	else:
	242	line = self.lines[self.index]
	243	self.index += 1
	244	return line
	245
	246	# Line-eater for tokenize.
	247	def tokeneater(self, type, token, (sline, scol), end, line,
	248	INDENT=tokenize.INDENT,
	249	DEDENT=tokenize.DEDENT,
	250	NEWLINE=tokenize.NEWLINE,
	251	COMMENT=tokenize.COMMENT,
	252	NL=tokenize.NL):
	253
	254	if type == NEWLINE:
	255	# A program statement, or ENDMARKER, will eventually follow,
	256	# after some (possibly empty) run of tokens of the form
	257	# (NL \| COMMENT)* (INDENT \| DEDENT+)?
	258	self.find_stmt = 1
	259
	260	elif type == INDENT:
	261	self.find_stmt = 1
	262	self.level += 1
	263
	264	elif type == DEDENT:
	265	self.find_stmt = 1
	266	self.level -= 1
	267
	268	elif type == COMMENT:
	269	if self.find_stmt:
	270	self.stats.append((sline, -1))
	271	# but we're still looking for a new stmt, so leave
	272	# find_stmt alone
	273
	274	elif type == NL:
	275	pass
	276
	277	elif self.find_stmt:
	278	# This is the first "real token" following a NEWLINE, so it
	279	# must be the first token of the next program statement, or an
	280	# ENDMARKER.
	281	self.find_stmt = 0
	282	if line: # not endmarker
	283	self.stats.append((sline, self.level))
	284
	285	# Count number of leading blanks.
	286	def getlspace(line):
	287	i, n = 0, len(line)
	288	while i < n and line[i] == " ":
	289	i += 1
	290	return i
	291
	292	if __name__ == '__main__':
	293	main()

sys/src/cmd/python/Tools/scripts/rgrep.py+64−0

@@ -0,0 +1,64 @@
	1	#! /usr/bin/env python
	2
	3	"""Reverse grep.
	4
	5	Usage: rgrep [-i] pattern file
	6	"""
	7
	8	import sys
	9	import re
	10	import getopt
	11
	12	def main():
	13	bufsize = 64*1024
	14	reflags = 0
	15	opts, args = getopt.getopt(sys.argv[1:], "i")
	16	for o, a in opts:
	17	if o == '-i':
	18	reflags = reflags \| re.IGNORECASE
	19	if len(args) < 2:
	20	usage("not enough arguments")
	21	if len(args) > 2:
	22	usage("exactly one file argument required")
	23	pattern, filename = args
	24	try:
	25	prog = re.compile(pattern, reflags)
	26	except re.error, msg:
	27	usage("error in regular expression: %s" % str(msg))
	28	try:
	29	f = open(filename)
	30	except IOError, msg:
	31	usage("can't open %s: %s" % (repr(filename), str(msg)), 1)
	32	f.seek(0, 2)
	33	pos = f.tell()
	34	leftover = None
	35	while pos > 0:
	36	size = min(pos, bufsize)
	37	pos = pos - size
	38	f.seek(pos)
	39	buffer = f.read(size)
	40	lines = buffer.split("\n")
	41	del buffer
	42	if leftover is None:
	43	if not lines[-1]:
	44	del lines[-1]
	45	else:
	46	lines[-1] = lines[-1] + leftover
	47	if pos > 0:
	48	leftover = lines[0]
	49	del lines[0]
	50	else:
	51	leftover = None
	52	lines.reverse()
	53	for line in lines:
	54	if prog.search(line):
	55	print line
	56
	57	def usage(msg, code=2):
	58	sys.stdout = sys.stderr
	59	print msg
	60	print __doc__
	61	sys.exit(code)
	62
	63	if __name__ == '__main__':
	64	main()

sys/src/cmd/python/Tools/scripts/setup.py+19−0

@@ -0,0 +1,19 @@
	1	from distutils.core import setup
	2
	3	if __name__ == '__main__':
	4	setup(
	5	scripts=[
	6	'byteyears.py',
	7	'checkpyc.py',
	8	'copytime.py',
	9	'crlf.py',
	10	'dutree.py',
	11	'ftpmirror.py',
	12	'h2py.py',
	13	'lfcr.py',
	14	'logmerge.py',
	15	'../../Lib/tabnanny.py',
	16	'../../Lib/timeit.py',
	17	'untabify.py',
	18	],
	19	)

sys/src/cmd/python/Tools/scripts/suff.py+30−0

@@ -0,0 +1,30 @@
	1	#! /usr/bin/env python
	2
	3	# suff
	4	#
	5	# show different suffixes amongst arguments
	6
	7	import sys
	8
	9	def main():
	10	files = sys.argv[1:]
	11	suffixes = {}
	12	for filename in files:
	13	suff = getsuffix(filename)
	14	if not suffixes.has_key(suff):
	15	suffixes[suff] = []
	16	suffixes[suff].append(filename)
	17	keys = suffixes.keys()
	18	keys.sort()
	19	for suff in keys:
	20	print repr(suff), len(suffixes[suff])
	21
	22	def getsuffix(filename):
	23	suff = ''
	24	for i in range(len(filename)):
	25	if filename[i] == '.':
	26	suff = filename[i:]
	27	return suff
	28
	29	if __name__ == '__main__':
	30	main()

sys/src/cmd/python/Tools/scripts/svneol.py+77−0

@@ -0,0 +1,77 @@
	1	#! /usr/bin/env python
	2
	3	"""
	4	SVN helper script.
	5
	6	Try to set the svn:eol-style property to "native" on every .py, .txt, .c and
	7	.h file in the directory tree rooted at the current directory.
	8
	9	Files with the svn:eol-style property already set (to anything) are skipped.
	10
	11	svn will itself refuse to set this property on a file that's not under SVN
	12	control, or that has a binary mime-type property set. This script inherits
	13	that behavior, and passes on whatever warning message the failing "svn
	14	propset" command produces.
	15
	16	In the Python project, it's safe to invoke this script from the root of
	17	a checkout.
	18
	19	No output is produced for files that are ignored. For a file that gets
	20	svn:eol-style set, output looks like:
	21
	22	property 'svn:eol-style' set on 'Lib\ctypes\__init__.py'
	23
	24	For a file not under version control:
	25
	26	svn: warning: 'patch-finalizer.txt' is not under version control
	27
	28	and for a file with a binary mime-type property:
	29
	30	svn: File 'Lib\test\test_pep263.py' has binary mime type property
	31	"""
	32
	33	import re
	34	import os
	35
	36	def proplist(root, fn):
	37	"Return a list of property names for file fn in directory root"
	38	path = os.path.join(root, ".svn", "props", fn+".svn-work")
	39	try:
	40	f = open(path)
	41	except IOError:
	42	# no properties file: not under version control
	43	return []
	44	result = []
	45	while 1:
	46	# key-value pairs, of the form
	47	# K <length>
	48	# <keyname>NL
	49	# V length
	50	# <value>NL
	51	# END
	52	line = f.readline()
	53	if line.startswith("END"):
	54	break
	55	assert line.startswith("K ")
	56	L = int(line.split()[1])
	57	key = f.read(L)
	58	result.append(key)
	59	f.readline()
	60	line = f.readline()
	61	assert line.startswith("V ")
	62	L = int(line.split()[1])
	63	value = f.read(L)
	64	f.readline()
	65	f.close()
	66	return result
	67
	68	possible_text_file = re.compile(r"\.([hc]\|py\|txt\|sln\|vcproj)$").search
	69
	70	for root, dirs, files in os.walk('.'):
	71	if '.svn' in dirs:
	72	dirs.remove('.svn')
	73	for fn in files:
	74	if possible_text_file(fn):
	75	if 'svn:eol-style' not in proplist(root, fn):
	76	path = os.path.join(root, fn)
	77	os.system('svn propset svn:eol-style native "%s"' % path)

sys/src/cmd/python/Tools/scripts/texcheck.py+233−0

@@ -0,0 +1,233 @@
	1	""" TeXcheck.py -- rough syntax checking on Python style LaTeX documents.
	2
	3	Written by Raymond D. Hettinger <python at rcn.com>
	4	Copyright (c) 2003 Python Software Foundation. All rights reserved.
	5
	6	Designed to catch common markup errors including:
	7	* Unbalanced or mismatched parenthesis, brackets, and braces.
	8	* Unbalanced or mismatched \\begin and \\end blocks.
	9	* Misspelled or invalid LaTeX commands.
	10	* Use of forward slashes instead of backslashes for commands.
	11	* Table line size mismatches.
	12
	13	Sample command line usage:
	14	python texcheck.py -k chapterheading -m lib/librandomtex *.tex
	15
	16	Options:
	17	-m Munge parenthesis and brackets. [0,n) would normally mismatch.
	18	-k keyword: Keyword is a valid LaTeX command. Do not include the backslash.
	19	-d: Delimiter check only (useful for non-LaTeX files).
	20	-h: Help
	21	-s lineno: Start at lineno (useful for skipping complex sections).
	22	-v: Verbose. Trace the matching of //begin and //end blocks.
	23	"""
	24
	25	import re
	26	import sys
	27	import getopt
	28	from itertools import izip, count, islice
	29	import glob
	30
	31	cmdstr = r"""
	32	\section \module \declaremodule \modulesynopsis \moduleauthor
	33	\sectionauthor \versionadded \code \class \method \begin
	34	\optional \var \ref \end \subsection \lineiii \hline \label
	35	\indexii \textrm \ldots \keyword \stindex \index \item \note
	36	\withsubitem \ttindex \footnote \citetitle \samp \opindex
	37	\noindent \exception \strong \dfn \ctype \obindex \character
	38	\indexiii \function \bifuncindex \refmodule \refbimodindex
	39	\subsubsection \nodename \member \chapter \emph \ASCII \UNIX
	40	\regexp \program \production \token \productioncont \term
	41	\grammartoken \lineii \seemodule \file \EOF \documentclass
	42	\usepackage \title \input \maketitle \ifhtml \fi \url \Cpp
	43	\tableofcontents \kbd \programopt \envvar \refstmodindex
	44	\cfunction \constant \NULL \moreargs \cfuncline \cdata
	45	\textasciicircum \n \ABC \setindexsubitem \versionchanged
	46	\deprecated \seetext \newcommand \POSIX \pep \warning \rfc
	47	\verbatiminput \methodline \textgreater \seetitle \lineiv
	48	\funclineni \ulink \manpage \funcline \dataline \unspecified
	49	\textbackslash \mimetype \mailheader \seepep \textunderscore
	50	\longprogramopt \infinity \plusminus \shortversion \version
	51	\refmodindex \seerfc \makeindex \makemodindex \renewcommand
	52	\indexname \appendix \protect \indexiv \mbox \textasciitilde
	53	\platform \seeurl \leftmargin \labelwidth \localmoduletable
	54	\LaTeX \copyright \memberline \backslash \pi \centerline
	55	\caption \vspace \textwidth \menuselection \textless
	56	\makevar \csimplemacro \menuselection \bfcode \sub \release
	57	\email \kwindex \refexmodindex \filenq \e \menuselection
	58	\exindex \linev \newsgroup \verbatim \setshortversion
	59	\author \authoraddress \paragraph \subparagraph \cmemberline
	60	\textbar \C \seelink
	61	"""
	62
	63	def matchclose(c_lineno, c_symbol, openers, pairmap):
	64	"Verify that closing delimiter matches most recent opening delimiter"
	65	try:
	66	o_lineno, o_symbol = openers.pop()
	67	except IndexError:
	68	print "\nDelimiter mismatch. On line %d, encountered closing '%s' without corresponding open" % (c_lineno, c_symbol)
	69	return
	70	if o_symbol in pairmap.get(c_symbol, [c_symbol]): return
	71	print "\nOpener '%s' on line %d was not closed before encountering '%s' on line %d" % (o_symbol, o_lineno, c_symbol, c_lineno)
	72	return
	73
	74	def checkit(source, opts, morecmds=[]):
	75	"""Check the LaTeX formatting in a sequence of lines.
	76
	77	Opts is a mapping of options to option values if any:
	78	-m munge parenthesis and brackets
	79	-d delimiters only checking
	80	-v verbose trace of delimiter matching
	81	-s lineno: linenumber to start scan (default is 1).
	82
	83	Morecmds is a sequence of LaTeX commands (without backslashes) that
	84	are to be considered valid in the scan.
	85	"""
	86
	87	texcmd = re.compile(r'\\[A-Za-z]+')
	88	falsetexcmd = re.compile(r'\/([A-Za-z]+)') # Mismarked with forward slash
	89
	90	validcmds = set(cmdstr.split())
	91	for cmd in morecmds:
	92	validcmds.add('\\' + cmd)
	93
	94	if '-m' in opts:
	95	pairmap = {']':'[(', ')':'(['} # Munged openers
	96	else:
	97	pairmap = {']':'[', ')':'('} # Normal opener for a given closer
	98	openpunct = set('([') # Set of valid openers
	99
	100	delimiters = re.compile(r'\\(begin\|end){([_a-zA-Z]+)}\|([()\[\]])')
	101	braces = re.compile(r'({)\|(})')
	102	doubledwords = re.compile(r'(\b[A-za-z]+\b) \b\1\b')
	103	spacingmarkup = re.compile(r'\\(ABC\|ASCII\|C\|Cpp\|EOF\|infinity\|NULL\|plusminus\|POSIX\|UNIX)\s')
	104
	105	openers = [] # Stack of pending open delimiters
	106	bracestack = [] # Stack of pending open braces
	107
	108	tablestart = re.compile(r'\\begin{(?:long)?table([iv]+)}')
	109	tableline = re.compile(r'\\line([iv]+){')
	110	tableend = re.compile(r'\\end{(?:long)?table([iv]+)}')
	111	tablelevel = ''
	112	tablestartline = 0
	113
	114	startline = int(opts.get('-s', '1'))
	115	lineno = 0
	116
	117	for lineno, line in izip(count(startline), islice(source, startline-1, None)):
	118	line = line.rstrip()
	119
	120	# Check balancing of open/close parenthesis, brackets, and begin/end blocks
	121	for begend, name, punct in delimiters.findall(line):
	122	if '-v' in opts:
	123	print lineno, '\|', begend, name, punct,
	124	if begend == 'begin' and '-d' not in opts:
	125	openers.append((lineno, name))
	126	elif punct in openpunct:
	127	openers.append((lineno, punct))
	128	elif begend == 'end' and '-d' not in opts:
	129	matchclose(lineno, name, openers, pairmap)
	130	elif punct in pairmap:
	131	matchclose(lineno, punct, openers, pairmap)
	132	if '-v' in opts:
	133	print ' --> ', openers
	134
	135	# Balance opening and closing braces
	136	for open, close in braces.findall(line):
	137	if open == '{':
	138	bracestack.append(lineno)
	139	if close == '}':
	140	try:
	141	bracestack.pop()
	142	except IndexError:
	143	print r'Warning, unmatched } on line %s.' % (lineno,)
	144
	145	# Optionally, skip LaTeX specific checks
	146	if '-d' in opts:
	147	continue
	148
	149	# Warn whenever forward slashes encountered with a LaTeX command
	150	for cmd in falsetexcmd.findall(line):
	151	if '822' in line or '.html' in line:
	152	continue # Ignore false positives for urls and for /rfc822
	153	if '\\' + cmd in validcmds:
	154	print 'Warning, forward slash used on line %d with cmd: /%s' % (lineno, cmd)
	155
	156	# Check for markup requiring {} for correct spacing
	157	for cmd in spacingmarkup.findall(line):
	158	print r'Warning, \%s should be written as \%s{} on line %d' % (cmd, cmd, lineno)
	159
	160	# Validate commands
	161	nc = line.find(r'\newcommand')
	162	if nc != -1:
	163	start = line.find('{', nc)
	164	end = line.find('}', start)
	165	validcmds.add(line[start+1:end])
	166	for cmd in texcmd.findall(line):
	167	if cmd not in validcmds:
	168	print r'Warning, unknown tex cmd on line %d: \%s' % (lineno, cmd)
	169
	170	# Check table levels (make sure lineii only inside tableii)
	171	m = tablestart.search(line)
	172	if m:
	173	tablelevel = m.group(1)
	174	tablestartline = lineno
	175	m = tableline.search(line)
	176	if m and m.group(1) != tablelevel:
	177	print r'Warning, \line%s on line %d does not match \table%s on line %d' % (m.group(1), lineno, tablelevel, tablestartline)
	178	if tableend.search(line):
	179	tablelevel = ''
	180
	181	# Style guide warnings
	182	if 'e.g.' in line or 'i.e.' in line:
	183	print r'Style warning, avoid use of i.e or e.g. on line %d' % (lineno,)
	184
	185	for dw in doubledwords.findall(line):
	186	print r'Doubled word warning. "%s" on line %d' % (dw, lineno)
	187
	188	lastline = lineno
	189	for lineno, symbol in openers:
	190	print "Unmatched open delimiter '%s' on line %d" % (symbol, lineno)
	191	for lineno in bracestack:
	192	print "Unmatched { on line %d" % (lineno,)
	193	print 'Done checking %d lines.' % (lastline,)
	194	return 0
	195
	196	def main(args=None):
	197	if args is None:
	198	args = sys.argv[1:]
	199	optitems, arglist = getopt.getopt(args, "k:mdhs:v")
	200	opts = dict(optitems)
	201	if '-h' in opts or args==[]:
	202	print __doc__
	203	return 0
	204
	205	if len(arglist) < 1:
	206	print 'Please specify a file to be checked'
	207	return 1
	208
	209	for i, filespec in enumerate(arglist):
	210	if '*' in filespec or '?' in filespec:
	211	arglist[i:i+1] = glob.glob(filespec)
	212
	213	morecmds = [v for k,v in optitems if k=='-k']
	214	err = []
	215
	216	for filename in arglist:
	217	print '=' * 30
	218	print "Checking", filename
	219	try:
	220	f = open(filename)
	221	except IOError:
	222	print 'Cannot open file %s.' % arglist[0]
	223	return 2
	224
	225	try:
	226	err.append(checkit(f, opts, morecmds))
	227	finally:
	228	f.close()
	229
	230	return max(err)
	231
	232	if __name__ == '__main__':
	233	sys.exit(main())

sys/src/cmd/python/Tools/scripts/texi2html.py+2078−0

@@ -0,0 +1,2078 @@
	1	#! /usr/bin/env python
	2
	3	# Convert GNU texinfo files into HTML, one file per node.
	4	# Based on Texinfo 2.14.
	5	# Usage: texi2html [-d] [-d] [-c] inputfile outputdirectory
	6	# The input file must be a complete texinfo file, e.g. emacs.texi.
	7	# This creates many files (one per info node) in the output directory,
	8	# overwriting existing files of the same name. All files created have
	9	# ".html" as their extension.
	10
	11
	12	# XXX To do:
	13	# - handle @comment*** correctly
	14	# - handle @xref {some words} correctly
	15	# - handle @ftable correctly (items aren't indexed?)
	16	# - handle @itemx properly
	17	# - handle @exdent properly
	18	# - add links directly to the proper line from indices
	19	# - check against the definitive list of @-cmds; we still miss (among others):
	20	# - @defindex (hard)
	21	# - @c(omment) in the middle of a line (rarely used)
	22	# - @this* (not really needed, only used in headers anyway)
	23	# - @today{} (ever used outside title page?)
	24
	25	# More consistent handling of chapters/sections/etc.
	26	# Lots of documentation
	27	# Many more options:
	28	# -top designate top node
	29	# -links customize which types of links are included
	30	# -split split at chapters or sections instead of nodes
	31	# -name Allow different types of filename handling. Non unix systems
	32	# will have problems with long node names
	33	# ...
	34	# Support the most recent texinfo version and take a good look at HTML 3.0
	35	# More debugging output (customizable) and more flexible error handling
	36	# How about icons ?
	37
	38	# rpyron 2002-05-07
	39	# Robert Pyron <rpyron@alum.mit.edu>
	40	# 1. BUGFIX: In function makefile(), strip blanks from the nodename.
	41	# This is necesary to match the behavior of parser.makeref() and
	42	# parser.do_node().
	43	# 2. BUGFIX fixed KeyError in end_ifset (well, I may have just made
	44	# it go away, rather than fix it)
	45	# 3. BUGFIX allow @menu and menu items inside @ifset or @ifclear
	46	# 4. Support added for:
	47	# @uref URL reference
	48	# @image image file reference (see note below)
	49	# @multitable output an HTML table
	50	# @vtable
	51	# 5. Partial support for accents, to match MAKEINFO output
	52	# 6. I added a new command-line option, '-H basename', to specify
	53	# HTML Help output. This will cause three files to be created
	54	# in the current directory:
	55	# `basename`.hhp HTML Help Workshop project file
	56	# `basename`.hhc Contents file for the project
	57	# `basename`.hhk Index file for the project
	58	# When fed into HTML Help Workshop, the resulting file will be
	59	# named `basename`.chm.
	60	# 7. A new class, HTMLHelp, to accomplish item 6.
	61	# 8. Various calls to HTMLHelp functions.
	62	# A NOTE ON IMAGES: Just as 'outputdirectory' must exist before
	63	# running this program, all referenced images must already exist
	64	# in outputdirectory.
	65
	66	import os
	67	import sys
	68	import string
	69	import re
	70
	71	MAGIC = '\\input texinfo'
	72
	73	cmprog = re.compile('^@([a-z]+)([ \t]\|$)') # Command (line-oriented)
	74	blprog = re.compile('^[ \t]*$') # Blank line
	75	kwprog = re.compile('@[a-z]+') # Keyword (embedded, usually
	76	# with {} args)
	77	spprog = re.compile('[\n@{}&<>]') # Special characters in
	78	# running text
	79	#
	80	# menu item (Yuck!)
	81	miprog = re.compile('^\* ([^:]):(:\|[ \t]([^\t,\n.]+)([^ \t\n]))[ \t\n]')
	82	# 0 1 1 2 3 34 42 0
	83	# ----- ---------- ---------
	84	# -\|-----------------------------
	85	# -----------------------------------------------------
	86
	87
	88
	89
	90	class HTMLNode:
	91	"""Some of the parser's functionality is separated into this class.
	92
	93	A Node accumulates its contents, takes care of links to other Nodes
	94	and saves itself when it is finished and all links are resolved.
	95	"""
	96
	97	DOCTYPE = '<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">'
	98
	99	type = 0
	100	cont = ''
	101	epilogue = '</BODY></HTML>\n'
	102
	103	def __init__(self, dir, name, topname, title, next, prev, up):
	104	self.dirname = dir
	105	self.name = name
	106	if topname:
	107	self.topname = topname
	108	else:
	109	self.topname = name
	110	self.title = title
	111	self.next = next
	112	self.prev = prev
	113	self.up = up
	114	self.lines = []
	115
	116	def write(self, *lines):
	117	map(self.lines.append, lines)
	118
	119	def flush(self):
	120	fp = open(self.dirname + '/' + makefile(self.name), 'w')
	121	fp.write(self.prologue)
	122	fp.write(self.text)
	123	fp.write(self.epilogue)
	124	fp.close()
	125
	126	def link(self, label, nodename, rel=None, rev=None):
	127	if nodename:
	128	if nodename.lower() == '(dir)':
	129	addr = '../dir.html'
	130	title = ''
	131	else:
	132	addr = makefile(nodename)
	133	title = ' TITLE="%s"' % nodename
	134	self.write(label, ': <A HREF="', addr, '"', \
	135	rel and (' REL=' + rel) or "", \
	136	rev and (' REV=' + rev) or "", \
	137	title, '>', nodename, '</A> \n')
	138
	139	def finalize(self):
	140	length = len(self.lines)
	141	self.text = ''.join(self.lines)
	142	self.lines = []
	143	self.open_links()
	144	self.output_links()
	145	self.close_links()
	146	links = ''.join(self.lines)
	147	self.lines = []
	148	self.prologue = (
	149	self.DOCTYPE +
	150	'\n<HTML><HEAD>\n'
	151	' <!-- Converted with texi2html and Python -->\n'
	152	' <TITLE>' + self.title + '</TITLE>\n'
	153	' <LINK REL=Next HREF="'
	154	+ makefile(self.next) + '" TITLE="' + self.next + '">\n'
	155	' <LINK REL=Previous HREF="'
	156	+ makefile(self.prev) + '" TITLE="' + self.prev + '">\n'
	157	' <LINK REL=Up HREF="'
	158	+ makefile(self.up) + '" TITLE="' + self.up + '">\n'
	159	'</HEAD><BODY>\n' +
	160	links)
	161	if length > 20:
	162	self.epilogue = '<P>\n%s</BODY></HTML>\n' % links
	163
	164	def open_links(self):
	165	self.write('<HR>\n')
	166
	167	def close_links(self):
	168	self.write('<HR>\n')
	169
	170	def output_links(self):
	171	if self.cont != self.next:
	172	self.link(' Cont', self.cont)
	173	self.link(' Next', self.next, rel='Next')
	174	self.link(' Prev', self.prev, rel='Previous')
	175	self.link(' Up', self.up, rel='Up')
	176	if self.name <> self.topname:
	177	self.link(' Top', self.topname)
	178
	179
	180	class HTML3Node(HTMLNode):
	181
	182	DOCTYPE = '<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML Level 3//EN//3.0">'
	183
	184	def open_links(self):
	185	self.write('<DIV CLASS=Navigation>\n <HR>\n')
	186
	187	def close_links(self):
	188	self.write(' <HR>\n</DIV>\n')
	189
	190
	191	class TexinfoParser:
	192
	193	COPYRIGHT_SYMBOL = "©"
	194	FN_ID_PATTERN = "(%(id)s)"
	195	FN_SOURCE_PATTERN = '<A NAME=footnoteref%(id)s' \
	196	' HREF="#footnotetext%(id)s">' \
	197	+ FN_ID_PATTERN + '</A>'
	198	FN_TARGET_PATTERN = '<A NAME=footnotetext%(id)s' \
	199	' HREF="#footnoteref%(id)s">' \
	200	+ FN_ID_PATTERN + '</A>\n%(text)s<P>\n'
	201	FN_HEADER = '\n<P>\n<HR NOSHADE SIZE=1 WIDTH=200>\n' \
	202	'<STRONG><EM>Footnotes</EM></STRONG>\n<P>'
	203
	204
	205	Node = HTMLNode
	206
	207	# Initialize an instance
	208	def __init__(self):
	209	self.unknown = {} # statistics about unknown @-commands
	210	self.filenames = {} # Check for identical filenames
	211	self.debugging = 0 # larger values produce more output
	212	self.print_headers = 0 # always print headers?
	213	self.nodefp = None # open file we're writing to
	214	self.nodelineno = 0 # Linenumber relative to node
	215	self.links = None # Links from current node
	216	self.savetext = None # If not None, save text head instead
	217	self.savestack = [] # If not None, save text head instead
	218	self.htmlhelp = None # html help data
	219	self.dirname = 'tmp' # directory where files are created
	220	self.includedir = '.' # directory to search @include files
	221	self.nodename = '' # name of current node
	222	self.topname = '' # name of top node (first node seen)
	223	self.title = '' # title of this whole Texinfo tree
	224	self.resetindex() # Reset all indices
	225	self.contents = [] # Reset table of contents
	226	self.numbering = [] # Reset section numbering counters
	227	self.nofill = 0 # Normal operation: fill paragraphs
	228	self.values={'html': 1} # Names that should be parsed in ifset
	229	self.stackinfo={} # Keep track of state in the stack
	230	# XXX The following should be reset per node?!
	231	self.footnotes = [] # Reset list of footnotes
	232	self.itemarg = None # Reset command used by @item
	233	self.itemnumber = None # Reset number for @item in @enumerate
	234	self.itemindex = None # Reset item index name
	235	self.node = None
	236	self.nodestack = []
	237	self.cont = 0
	238	self.includedepth = 0
	239
	240	# Set htmlhelp helper class
	241	def sethtmlhelp(self, htmlhelp):
	242	self.htmlhelp = htmlhelp
	243
	244	# Set (output) directory name
	245	def setdirname(self, dirname):
	246	self.dirname = dirname
	247
	248	# Set include directory name
	249	def setincludedir(self, includedir):
	250	self.includedir = includedir
	251
	252	# Parse the contents of an entire file
	253	def parse(self, fp):
	254	line = fp.readline()
	255	lineno = 1
	256	while line and (line[0] == '%' or blprog.match(line)):
	257	line = fp.readline()
	258	lineno = lineno + 1
	259	if line[:len(MAGIC)] <> MAGIC:
	260	raise SyntaxError, 'file does not begin with %r' % (MAGIC,)
	261	self.parserest(fp, lineno)
	262
	263	# Parse the contents of a file, not expecting a MAGIC header
	264	def parserest(self, fp, initial_lineno):
	265	lineno = initial_lineno
	266	self.done = 0
	267	self.skip = 0
	268	self.stack = []
	269	accu = []
	270	while not self.done:
	271	line = fp.readline()
	272	self.nodelineno = self.nodelineno + 1
	273	if not line:
	274	if accu:
	275	if not self.skip: self.process(accu)
	276	accu = []
	277	if initial_lineno > 0:
	278	print '*** EOF before @bye'
	279	break
	280	lineno = lineno + 1
	281	mo = cmprog.match(line)
	282	if mo:
	283	a, b = mo.span(1)
	284	cmd = line[a:b]
	285	if cmd in ('noindent', 'refill'):
	286	accu.append(line)
	287	else:
	288	if accu:
	289	if not self.skip:
	290	self.process(accu)
	291	accu = []
	292	self.command(line, mo)
	293	elif blprog.match(line) and \
	294	'format' not in self.stack and \
	295	'example' not in self.stack:
	296	if accu:
	297	if not self.skip:
	298	self.process(accu)
	299	if self.nofill:
	300	self.write('\n')
	301	else:
	302	self.write('<P>\n')
	303	accu = []
	304	else:
	305	# Append the line including trailing \n!
	306	accu.append(line)
	307	#
	308	if self.skip:
	309	print '*** Still skipping at the end'
	310	if self.stack:
	311	print '*** Stack not empty at the end'
	312	print '***', self.stack
	313	if self.includedepth == 0:
	314	while self.nodestack:
	315	self.nodestack[-1].finalize()
	316	self.nodestack[-1].flush()
	317	del self.nodestack[-1]
	318
	319	# Start saving text in a buffer instead of writing it to a file
	320	def startsaving(self):
	321	if self.savetext <> None:
	322	self.savestack.append(self.savetext)
	323	# print '*** Recursively saving text, expect trouble'
	324	self.savetext = ''
	325
	326	# Return the text saved so far and start writing to file again
	327	def collectsavings(self):
	328	savetext = self.savetext
	329	if len(self.savestack) > 0:
	330	self.savetext = self.savestack[-1]
	331	del self.savestack[-1]
	332	else:
	333	self.savetext = None
	334	return savetext or ''
	335
	336	# Write text to file, or save it in a buffer, or ignore it
	337	def write(self, *args):
	338	try:
	339	text = ''.join(args)
	340	except:
	341	print args
	342	raise TypeError
	343	if self.savetext <> None:
	344	self.savetext = self.savetext + text
	345	elif self.nodefp:
	346	self.nodefp.write(text)
	347	elif self.node:
	348	self.node.write(text)
	349
	350	# Complete the current node -- write footnotes and close file
	351	def endnode(self):
	352	if self.savetext <> None:
	353	print '*** Still saving text at end of node'
	354	dummy = self.collectsavings()
	355	if self.footnotes:
	356	self.writefootnotes()
	357	if self.nodefp:
	358	if self.nodelineno > 20:
	359	self.write('<HR>\n')
	360	[name, next, prev, up] = self.nodelinks[:4]
	361	self.link('Next', next)
	362	self.link('Prev', prev)
	363	self.link('Up', up)
	364	if self.nodename <> self.topname:
	365	self.link('Top', self.topname)
	366	self.write('<HR>\n')
	367	self.write('</BODY>\n')
	368	self.nodefp.close()
	369	self.nodefp = None
	370	elif self.node:
	371	if not self.cont and \
	372	(not self.node.type or \
	373	(self.node.next and self.node.prev and self.node.up)):
	374	self.node.finalize()
	375	self.node.flush()
	376	else:
	377	self.nodestack.append(self.node)
	378	self.node = None
	379	self.nodename = ''
	380
	381	# Process a list of lines, expanding embedded @-commands
	382	# This mostly distinguishes between menus and normal text
	383	def process(self, accu):
	384	if self.debugging > 1:
	385	print '!'*self.debugging, 'process:', self.skip, self.stack,
	386	if accu: print accu[0][:30],
	387	if accu[0][30:] or accu[1:]: print '...',
	388	print
	389	if self.inmenu():
	390	# XXX should be done differently
	391	for line in accu:
	392	mo = miprog.match(line)
	393	if not mo:
	394	line = line.strip() + '\n'
	395	self.expand(line)
	396	continue
	397	bgn, end = mo.span(0)
	398	a, b = mo.span(1)
	399	c, d = mo.span(2)
	400	e, f = mo.span(3)
	401	g, h = mo.span(4)
	402	label = line[a:b]
	403	nodename = line[c:d]
	404	if nodename[0] == ':': nodename = label
	405	else: nodename = line[e:f]
	406	punct = line[g:h]
	407	self.write(' <LI><A HREF="',
	408	makefile(nodename),
	409	'">', nodename,
	410	'</A>', punct, '\n')
	411	self.htmlhelp.menuitem(nodename)
	412	self.expand(line[end:])
	413	else:
	414	text = ''.join(accu)
	415	self.expand(text)
	416
	417	# find 'menu' (we might be inside 'ifset' or 'ifclear')
	418	def inmenu(self):
	419	#if 'menu' in self.stack:
	420	# print 'inmenu :', self.skip, self.stack, self.stackinfo
	421	stack = self.stack
	422	while stack and stack[-1] in ('ifset','ifclear'):
	423	try:
	424	if self.stackinfo[len(stack)]:
	425	return 0
	426	except KeyError:
	427	pass
	428	stack = stack[:-1]
	429	return (stack and stack[-1] == 'menu')
	430
	431	# Write a string, expanding embedded @-commands
	432	def expand(self, text):
	433	stack = []
	434	i = 0
	435	n = len(text)
	436	while i < n:
	437	start = i
	438	mo = spprog.search(text, i)
	439	if mo:
	440	i = mo.start()
	441	else:
	442	self.write(text[start:])
	443	break
	444	self.write(text[start:i])
	445	c = text[i]
	446	i = i+1
	447	if c == '\n':
	448	self.write('\n')
	449	continue
	450	if c == '<':
	451	self.write('<')
	452	continue
	453	if c == '>':
	454	self.write('>')
	455	continue
	456	if c == '&':
	457	self.write('&')
	458	continue
	459	if c == '{':
	460	stack.append('')
	461	continue
	462	if c == '}':
	463	if not stack:
	464	print '*** Unmatched }'
	465	self.write('}')
	466	continue
	467	cmd = stack[-1]
	468	del stack[-1]
	469	try:
	470	method = getattr(self, 'close_' + cmd)
	471	except AttributeError:
	472	self.unknown_close(cmd)
	473	continue
	474	method()
	475	continue
	476	if c <> '@':
	477	# Cannot happen unless spprog is changed
	478	raise RuntimeError, 'unexpected funny %r' % c
	479	start = i
	480	while i < n and text[i] in string.ascii_letters: i = i+1
	481	if i == start:
	482	# @ plus non-letter: literal next character
	483	i = i+1
	484	c = text[start:i]
	485	if c == ':':
	486	# `@:' means no extra space after
	487	# preceding `.', `?', `!' or `:'
	488	pass
	489	else:
	490	# `@.' means a sentence-ending period;
	491	# `@@', `@{', `@}' quote `@', `{', `}'
	492	self.write(c)
	493	continue
	494	cmd = text[start:i]
	495	if i < n and text[i] == '{':
	496	i = i+1
	497	stack.append(cmd)
	498	try:
	499	method = getattr(self, 'open_' + cmd)
	500	except AttributeError:
	501	self.unknown_open(cmd)
	502	continue
	503	method()
	504	continue
	505	try:
	506	method = getattr(self, 'handle_' + cmd)
	507	except AttributeError:
	508	self.unknown_handle(cmd)
	509	continue
	510	method()
	511	if stack:
	512	print '*** Stack not empty at para:', stack
	513
	514	# --- Handle unknown embedded @-commands ---
	515
	516	def unknown_open(self, cmd):
	517	print '*** No open func for @' + cmd + '{...}'
	518	cmd = cmd + '{'
	519	self.write('@', cmd)
	520	if not self.unknown.has_key(cmd):
	521	self.unknown[cmd] = 1
	522	else:
	523	self.unknown[cmd] = self.unknown[cmd] + 1
	524
	525	def unknown_close(self, cmd):
	526	print '*** No close func for @' + cmd + '{...}'
	527	cmd = '}' + cmd
	528	self.write('}')
	529	if not self.unknown.has_key(cmd):
	530	self.unknown[cmd] = 1
	531	else:
	532	self.unknown[cmd] = self.unknown[cmd] + 1
	533
	534	def unknown_handle(self, cmd):
	535	print '*** No handler for @' + cmd
	536	self.write('@', cmd)
	537	if not self.unknown.has_key(cmd):
	538	self.unknown[cmd] = 1
	539	else:
	540	self.unknown[cmd] = self.unknown[cmd] + 1
	541
	542	# XXX The following sections should be ordered as the texinfo docs
	543
	544	# --- Embedded @-commands without {} argument list --
	545
	546	def handle_noindent(self): pass
	547
	548	def handle_refill(self): pass
	549
	550	# --- Include file handling ---
	551
	552	def do_include(self, args):
	553	file = args
	554	file = os.path.join(self.includedir, file)
	555	try:
	556	fp = open(file, 'r')
	557	except IOError, msg:
	558	print '*** Can\'t open include file', repr(file)
	559	return
	560	print '!'*self.debugging, '--> file', repr(file)
	561	save_done = self.done
	562	save_skip = self.skip
	563	save_stack = self.stack
	564	self.includedepth = self.includedepth + 1
	565	self.parserest(fp, 0)
	566	self.includedepth = self.includedepth - 1
	567	fp.close()
	568	self.done = save_done
	569	self.skip = save_skip
	570	self.stack = save_stack
	571	print '!'*self.debugging, '<-- file', repr(file)
	572
	573	# --- Special Insertions ---
	574
	575	def open_dmn(self): pass
	576	def close_dmn(self): pass
	577
	578	def open_dots(self): self.write('...')
	579	def close_dots(self): pass
	580
	581	def open_bullet(self): pass
	582	def close_bullet(self): pass
	583
	584	def open_TeX(self): self.write('TeX')
	585	def close_TeX(self): pass
	586
	587	def handle_copyright(self): self.write(self.COPYRIGHT_SYMBOL)
	588	def open_copyright(self): self.write(self.COPYRIGHT_SYMBOL)
	589	def close_copyright(self): pass
	590
	591	def open_minus(self): self.write('-')
	592	def close_minus(self): pass
	593
	594	# --- Accents ---
	595
	596	# rpyron 2002-05-07
	597	# I would like to do at least as well as makeinfo when
	598	# it is producing HTML output:
	599	#
	600	# input output
	601	# @"o @"o umlaut accent
	602	# @'o 'o acute accent
	603	# @,{c} @,{c} cedilla accent
	604	# @=o @=o macron/overbar accent
	605	# @^o @^o circumflex accent
	606	# @`o `o grave accent
	607	# @~o @~o tilde accent
	608	# @dotaccent{o} @dotaccent{o} overdot accent
	609	# @H{o} @H{o} long Hungarian umlaut
	610	# @ringaccent{o} @ringaccent{o} ring accent
	611	# @tieaccent{oo} @tieaccent{oo} tie-after accent
	612	# @u{o} @u{o} breve accent
	613	# @ubaraccent{o} @ubaraccent{o} underbar accent
	614	# @udotaccent{o} @udotaccent{o} underdot accent
	615	# @v{o} @v{o} hacek or check accent
	616	# @exclamdown{} ¡ upside-down !
	617	# @questiondown{} ¿ upside-down ?
	618	# @aa{},@AA{} å,Å a,A with circle
	619	# @ae{},@AE{} æ,Æ ae,AE ligatures
	620	# @dotless{i} @dotless{i} dotless i
	621	# @dotless{j} @dotless{j} dotless j
	622	# @l{},@L{} l/,L/ suppressed-L,l
	623	# @o{},@O{} ø,Ø O,o with slash
	624	# @oe{},@OE{} oe,OE oe,OE ligatures
	625	# @ss{} ß es-zet or sharp S
	626	#
	627	# The following character codes and approximations have been
	628	# copied from makeinfo's HTML output.
	629
	630	def open_exclamdown(self): self.write('¡') # upside-down !
	631	def close_exclamdown(self): pass
	632	def open_questiondown(self): self.write('¿') # upside-down ?
	633	def close_questiondown(self): pass
	634	def open_aa(self): self.write('å') # a with circle
	635	def close_aa(self): pass
	636	def open_AA(self): self.write('Å') # A with circle
	637	def close_AA(self): pass
	638	def open_ae(self): self.write('æ') # ae ligatures
	639	def close_ae(self): pass
	640	def open_AE(self): self.write('Æ') # AE ligatures
	641	def close_AE(self): pass
	642	def open_o(self): self.write('ø') # o with slash
	643	def close_o(self): pass
	644	def open_O(self): self.write('Ø') # O with slash
	645	def close_O(self): pass
	646	def open_ss(self): self.write('ß') # es-zet or sharp S
	647	def close_ss(self): pass
	648	def open_oe(self): self.write('oe') # oe ligatures
	649	def close_oe(self): pass
	650	def open_OE(self): self.write('OE') # OE ligatures
	651	def close_OE(self): pass
	652	def open_l(self): self.write('l/') # suppressed-l
	653	def close_l(self): pass
	654	def open_L(self): self.write('L/') # suppressed-L
	655	def close_L(self): pass
	656
	657	# --- Special Glyphs for Examples ---
	658
	659	def open_result(self): self.write('=>')
	660	def close_result(self): pass
	661
	662	def open_expansion(self): self.write('==>')
	663	def close_expansion(self): pass
	664
	665	def open_print(self): self.write('-\|')
	666	def close_print(self): pass
	667
	668	def open_error(self): self.write('error-->')
	669	def close_error(self): pass
	670
	671	def open_equiv(self): self.write('==')
	672	def close_equiv(self): pass
	673
	674	def open_point(self): self.write('-!-')
	675	def close_point(self): pass
	676
	677	# --- Cross References ---
	678
	679	def open_pxref(self):
	680	self.write('see ')
	681	self.startsaving()
	682	def close_pxref(self):
	683	self.makeref()
	684
	685	def open_xref(self):
	686	self.write('See ')
	687	self.startsaving()
	688	def close_xref(self):
	689	self.makeref()
	690
	691	def open_ref(self):
	692	self.startsaving()
	693	def close_ref(self):
	694	self.makeref()
	695
	696	def open_inforef(self):
	697	self.write('See info file ')
	698	self.startsaving()
	699	def close_inforef(self):
	700	text = self.collectsavings()
	701	args = [s.strip() for s in text.split(',')]
	702	while len(args) < 3: args.append('')
	703	node = args[0]
	704	file = args[2]
	705	self.write('`', file, '\', node `', node, '\'')
	706
	707	def makeref(self):
	708	text = self.collectsavings()
	709	args = [s.strip() for s in text.split(',')]
	710	while len(args) < 5: args.append('')
	711	nodename = label = args[0]
	712	if args[2]: label = args[2]
	713	file = args[3]
	714	title = args[4]
	715	href = makefile(nodename)
	716	if file:
	717	href = '../' + file + '/' + href
	718	self.write('<A HREF="', href, '">', label, '</A>')
	719
	720	# rpyron 2002-05-07 uref support
	721	def open_uref(self):
	722	self.startsaving()
	723	def close_uref(self):
	724	text = self.collectsavings()
	725	args = [s.strip() for s in text.split(',')]
	726	while len(args) < 2: args.append('')
	727	href = args[0]
	728	label = args[1]
	729	if not label: label = href
	730	self.write('<A HREF="', href, '">', label, '</A>')
	731
	732	# rpyron 2002-05-07 image support
	733	# GNU makeinfo producing HTML output tries `filename.png'; if
	734	# that does not exist, it tries `filename.jpg'. If that does
	735	# not exist either, it complains. GNU makeinfo does not handle
	736	# GIF files; however, I include GIF support here because
	737	# MySQL documentation uses GIF files.
	738
	739	def open_image(self):
	740	self.startsaving()
	741	def close_image(self):
	742	self.makeimage()
	743	def makeimage(self):
	744	text = self.collectsavings()
	745	args = [s.strip() for s in text.split(',')]
	746	while len(args) < 5: args.append('')
	747	filename = args[0]
	748	width = args[1]
	749	height = args[2]
	750	alt = args[3]
	751	ext = args[4]
	752
	753	# The HTML output will have a reference to the image
	754	# that is relative to the HTML output directory,
	755	# which is what 'filename' gives us. However, we need
	756	# to find it relative to our own current directory,
	757	# so we construct 'imagename'.
	758	imagelocation = self.dirname + '/' + filename
	759
	760	if os.path.exists(imagelocation+'.png'):
	761	filename += '.png'
	762	elif os.path.exists(imagelocation+'.jpg'):
	763	filename += '.jpg'
	764	elif os.path.exists(imagelocation+'.gif'): # MySQL uses GIF files
	765	filename += '.gif'
	766	else:
	767	print "*** Cannot find image " + imagelocation
	768	#TODO: what is 'ext'?
	769	self.write('<IMG SRC="', filename, '"', \
	770	width and (' WIDTH="' + width + '"') or "", \
	771	height and (' HEIGHT="' + height + '"') or "", \
	772	alt and (' ALT="' + alt + '"') or "", \
	773	'/>' )
	774	self.htmlhelp.addimage(imagelocation)
	775
	776
	777	# --- Marking Words and Phrases ---
	778
	779	# --- Other @xxx{...} commands ---
	780
	781	def open_(self): pass # Used by {text enclosed in braces}
	782	def close_(self): pass
	783
	784	open_asis = open_
	785	close_asis = close_
	786
	787	def open_cite(self): self.write('<CITE>')
	788	def close_cite(self): self.write('</CITE>')
	789
	790	def open_code(self): self.write('<CODE>')
	791	def close_code(self): self.write('</CODE>')
	792
	793	def open_t(self): self.write('<TT>')
	794	def close_t(self): self.write('</TT>')
	795
	796	def open_dfn(self): self.write('<DFN>')
	797	def close_dfn(self): self.write('</DFN>')
	798
	799	def open_emph(self): self.write('<EM>')
	800	def close_emph(self): self.write('</EM>')
	801
	802	def open_i(self): self.write('<I>')
	803	def close_i(self): self.write('</I>')
	804
	805	def open_footnote(self):
	806	# if self.savetext <> None:
	807	# print '*** Recursive footnote -- expect weirdness'
	808	id = len(self.footnotes) + 1
	809	self.write(self.FN_SOURCE_PATTERN % {'id': repr(id)})
	810	self.startsaving()
	811
	812	def close_footnote(self):
	813	id = len(self.footnotes) + 1
	814	self.footnotes.append((id, self.collectsavings()))
	815
	816	def writefootnotes(self):
	817	self.write(self.FN_HEADER)
	818	for id, text in self.footnotes:
	819	self.write(self.FN_TARGET_PATTERN
	820	% {'id': repr(id), 'text': text})
	821	self.footnotes = []
	822
	823	def open_file(self): self.write('<CODE>')
	824	def close_file(self): self.write('</CODE>')
	825
	826	def open_kbd(self): self.write('<KBD>')
	827	def close_kbd(self): self.write('</KBD>')
	828
	829	def open_key(self): self.write('<KEY>')
	830	def close_key(self): self.write('</KEY>')
	831
	832	def open_r(self): self.write('<R>')
	833	def close_r(self): self.write('</R>')
	834
	835	def open_samp(self): self.write('`<SAMP>')
	836	def close_samp(self): self.write('</SAMP>\'')
	837
	838	def open_sc(self): self.write('<SMALLCAPS>')
	839	def close_sc(self): self.write('</SMALLCAPS>')
	840
	841	def open_strong(self): self.write('<STRONG>')
	842	def close_strong(self): self.write('</STRONG>')
	843
	844	def open_b(self): self.write('<B>')
	845	def close_b(self): self.write('</B>')
	846
	847	def open_var(self): self.write('<VAR>')
	848	def close_var(self): self.write('</VAR>')
	849
	850	def open_w(self): self.write('<NOBREAK>')
	851	def close_w(self): self.write('</NOBREAK>')
	852
	853	def open_url(self): self.startsaving()
	854	def close_url(self):
	855	text = self.collectsavings()
	856	self.write('<A HREF="', text, '">', text, '</A>')
	857
	858	def open_email(self): self.startsaving()
	859	def close_email(self):
	860	text = self.collectsavings()
	861	self.write('<A HREF="mailto:', text, '">', text, '</A>')
	862
	863	open_titlefont = open_
	864	close_titlefont = close_
	865
	866	def open_small(self): pass
	867	def close_small(self): pass
	868
	869	def command(self, line, mo):
	870	a, b = mo.span(1)
	871	cmd = line[a:b]
	872	args = line[b:].strip()
	873	if self.debugging > 1:
	874	print '!'*self.debugging, 'command:', self.skip, self.stack, \
	875	'@' + cmd, args
	876	try:
	877	func = getattr(self, 'do_' + cmd)
	878	except AttributeError:
	879	try:
	880	func = getattr(self, 'bgn_' + cmd)
	881	except AttributeError:
	882	# don't complain if we are skipping anyway
	883	if not self.skip:
	884	self.unknown_cmd(cmd, args)
	885	return
	886	self.stack.append(cmd)
	887	func(args)
	888	return
	889	if not self.skip or cmd == 'end':
	890	func(args)
	891
	892	def unknown_cmd(self, cmd, args):
	893	print '*** unknown', '@' + cmd, args
	894	if not self.unknown.has_key(cmd):
	895	self.unknown[cmd] = 1
	896	else:
	897	self.unknown[cmd] = self.unknown[cmd] + 1
	898
	899	def do_end(self, args):
	900	words = args.split()
	901	if not words:
	902	print '*** @end w/o args'
	903	else:
	904	cmd = words[0]
	905	if not self.stack or self.stack[-1] <> cmd:
	906	print '*** @end', cmd, 'unexpected'
	907	else:
	908	del self.stack[-1]
	909	try:
	910	func = getattr(self, 'end_' + cmd)
	911	except AttributeError:
	912	self.unknown_end(cmd)
	913	return
	914	func()
	915
	916	def unknown_end(self, cmd):
	917	cmd = 'end ' + cmd
	918	print '*** unknown', '@' + cmd
	919	if not self.unknown.has_key(cmd):
	920	self.unknown[cmd] = 1
	921	else:
	922	self.unknown[cmd] = self.unknown[cmd] + 1
	923
	924	# --- Comments ---
	925
	926	def do_comment(self, args): pass
	927	do_c = do_comment
	928
	929	# --- Conditional processing ---
	930
	931	def bgn_ifinfo(self, args): pass
	932	def end_ifinfo(self): pass
	933
	934	def bgn_iftex(self, args): self.skip = self.skip + 1
	935	def end_iftex(self): self.skip = self.skip - 1
	936
	937	def bgn_ignore(self, args): self.skip = self.skip + 1
	938	def end_ignore(self): self.skip = self.skip - 1
	939
	940	def bgn_tex(self, args): self.skip = self.skip + 1
	941	def end_tex(self): self.skip = self.skip - 1
	942
	943	def do_set(self, args):
	944	fields = args.split(' ')
	945	key = fields[0]
	946	if len(fields) == 1:
	947	value = 1
	948	else:
	949	value = ' '.join(fields[1:])
	950	self.values[key] = value
	951
	952	def do_clear(self, args):
	953	self.values[args] = None
	954
	955	def bgn_ifset(self, args):
	956	if args not in self.values.keys() \
	957	or self.values[args] is None:
	958	self.skip = self.skip + 1
	959	self.stackinfo[len(self.stack)] = 1
	960	else:
	961	self.stackinfo[len(self.stack)] = 0
	962	def end_ifset(self):
	963	try:
	964	if self.stackinfo[len(self.stack) + 1]:
	965	self.skip = self.skip - 1
	966	del self.stackinfo[len(self.stack) + 1]
	967	except KeyError:
	968	print '*** end_ifset: KeyError :', len(self.stack) + 1
	969
	970	def bgn_ifclear(self, args):
	971	if args in self.values.keys() \
	972	and self.values[args] is not None:
	973	self.skip = self.skip + 1
	974	self.stackinfo[len(self.stack)] = 1
	975	else:
	976	self.stackinfo[len(self.stack)] = 0
	977	def end_ifclear(self):
	978	try:
	979	if self.stackinfo[len(self.stack) + 1]:
	980	self.skip = self.skip - 1
	981	del self.stackinfo[len(self.stack) + 1]
	982	except KeyError:
	983	print '*** end_ifclear: KeyError :', len(self.stack) + 1
	984
	985	def open_value(self):
	986	self.startsaving()
	987
	988	def close_value(self):
	989	key = self.collectsavings()
	990	if key in self.values.keys():
	991	self.write(self.values[key])
	992	else:
	993	print '*** Undefined value: ', key
	994
	995	# --- Beginning a file ---
	996
	997	do_finalout = do_comment
	998	do_setchapternewpage = do_comment
	999	do_setfilename = do_comment
	1000
	1001	def do_settitle(self, args):
	1002	self.startsaving()
	1003	self.expand(args)
	1004	self.title = self.collectsavings()
	1005	def do_parskip(self, args): pass
	1006
	1007	# --- Ending a file ---
	1008
	1009	def do_bye(self, args):
	1010	self.endnode()
	1011	self.done = 1
	1012
	1013	# --- Title page ---
	1014
	1015	def bgn_titlepage(self, args): self.skip = self.skip + 1
	1016	def end_titlepage(self): self.skip = self.skip - 1
	1017	def do_shorttitlepage(self, args): pass
	1018
	1019	def do_center(self, args):
	1020	# Actually not used outside title page...
	1021	self.write('<H1>')
	1022	self.expand(args)
	1023	self.write('</H1>\n')
	1024	do_title = do_center
	1025	do_subtitle = do_center
	1026	do_author = do_center
	1027
	1028	do_vskip = do_comment
	1029	do_vfill = do_comment
	1030	do_smallbook = do_comment
	1031
	1032	do_paragraphindent = do_comment
	1033	do_setchapternewpage = do_comment
	1034	do_headings = do_comment
	1035	do_footnotestyle = do_comment
	1036
	1037	do_evenheading = do_comment
	1038	do_evenfooting = do_comment
	1039	do_oddheading = do_comment
	1040	do_oddfooting = do_comment
	1041	do_everyheading = do_comment
	1042	do_everyfooting = do_comment
	1043
	1044	# --- Nodes ---
	1045
	1046	def do_node(self, args):
	1047	self.endnode()
	1048	self.nodelineno = 0
	1049	parts = [s.strip() for s in args.split(',')]
	1050	while len(parts) < 4: parts.append('')
	1051	self.nodelinks = parts
	1052	[name, next, prev, up] = parts[:4]
	1053	file = self.dirname + '/' + makefile(name)
	1054	if self.filenames.has_key(file):
	1055	print '*** Filename already in use: ', file
	1056	else:
	1057	if self.debugging: print '!'*self.debugging, '--- writing', file
	1058	self.filenames[file] = 1
	1059	# self.nodefp = open(file, 'w')
	1060	self.nodename = name
	1061	if self.cont and self.nodestack:
	1062	self.nodestack[-1].cont = self.nodename
	1063	if not self.topname: self.topname = name
	1064	title = name
	1065	if self.title: title = title + ' -- ' + self.title
	1066	self.node = self.Node(self.dirname, self.nodename, self.topname,
	1067	title, next, prev, up)
	1068	self.htmlhelp.addnode(self.nodename,next,prev,up,file)
	1069
	1070	def link(self, label, nodename):
	1071	if nodename:
	1072	if nodename.lower() == '(dir)':
	1073	addr = '../dir.html'
	1074	else:
	1075	addr = makefile(nodename)
	1076	self.write(label, ': <A HREF="', addr, '" TYPE="',
	1077	label, '">', nodename, '</A> \n')
	1078
	1079	# --- Sectioning commands ---
	1080
	1081	def popstack(self, type):
	1082	if (self.node):
	1083	self.node.type = type
	1084	while self.nodestack:
	1085	if self.nodestack[-1].type > type:
	1086	self.nodestack[-1].finalize()
	1087	self.nodestack[-1].flush()
	1088	del self.nodestack[-1]
	1089	elif self.nodestack[-1].type == type:
	1090	if not self.nodestack[-1].next:
	1091	self.nodestack[-1].next = self.node.name
	1092	if not self.node.prev:
	1093	self.node.prev = self.nodestack[-1].name
	1094	self.nodestack[-1].finalize()
	1095	self.nodestack[-1].flush()
	1096	del self.nodestack[-1]
	1097	else:
	1098	if type > 1 and not self.node.up:
	1099	self.node.up = self.nodestack[-1].name
	1100	break
	1101
	1102	def do_chapter(self, args):
	1103	self.heading('H1', args, 0)
	1104	self.popstack(1)
	1105
	1106	def do_unnumbered(self, args):
	1107	self.heading('H1', args, -1)
	1108	self.popstack(1)
	1109	def do_appendix(self, args):
	1110	self.heading('H1', args, -1)
	1111	self.popstack(1)
	1112	def do_top(self, args):
	1113	self.heading('H1', args, -1)
	1114	def do_chapheading(self, args):
	1115	self.heading('H1', args, -1)
	1116	def do_majorheading(self, args):
	1117	self.heading('H1', args, -1)
	1118
	1119	def do_section(self, args):
	1120	self.heading('H1', args, 1)
	1121	self.popstack(2)
	1122
	1123	def do_unnumberedsec(self, args):
	1124	self.heading('H1', args, -1)
	1125	self.popstack(2)
	1126	def do_appendixsec(self, args):
	1127	self.heading('H1', args, -1)
	1128	self.popstack(2)
	1129	do_appendixsection = do_appendixsec
	1130	def do_heading(self, args):
	1131	self.heading('H1', args, -1)
	1132
	1133	def do_subsection(self, args):
	1134	self.heading('H2', args, 2)
	1135	self.popstack(3)
	1136	def do_unnumberedsubsec(self, args):
	1137	self.heading('H2', args, -1)
	1138	self.popstack(3)
	1139	def do_appendixsubsec(self, args):
	1140	self.heading('H2', args, -1)
	1141	self.popstack(3)
	1142	def do_subheading(self, args):
	1143	self.heading('H2', args, -1)
	1144
	1145	def do_subsubsection(self, args):
	1146	self.heading('H3', args, 3)
	1147	self.popstack(4)
	1148	def do_unnumberedsubsubsec(self, args):
	1149	self.heading('H3', args, -1)
	1150	self.popstack(4)
	1151	def do_appendixsubsubsec(self, args):
	1152	self.heading('H3', args, -1)
	1153	self.popstack(4)
	1154	def do_subsubheading(self, args):
	1155	self.heading('H3', args, -1)
	1156
	1157	def heading(self, type, args, level):
	1158	if level >= 0:
	1159	while len(self.numbering) <= level:
	1160	self.numbering.append(0)
	1161	del self.numbering[level+1:]
	1162	self.numbering[level] = self.numbering[level] + 1
	1163	x = ''
	1164	for i in self.numbering:
	1165	x = x + repr(i) + '.'
	1166	args = x + ' ' + args
	1167	self.contents.append((level, args, self.nodename))
	1168	self.write('<', type, '>')
	1169	self.expand(args)
	1170	self.write('</', type, '>\n')
	1171	if self.debugging or self.print_headers:
	1172	print '---', args
	1173
	1174	def do_contents(self, args):
	1175	# pass
	1176	self.listcontents('Table of Contents', 999)
	1177
	1178	def do_shortcontents(self, args):
	1179	pass
	1180	# self.listcontents('Short Contents', 0)
	1181	do_summarycontents = do_shortcontents
	1182
	1183	def listcontents(self, title, maxlevel):
	1184	self.write('<H1>', title, '</H1>\n<UL COMPACT PLAIN>\n')
	1185	prevlevels = [0]
	1186	for level, title, node in self.contents:
	1187	if level > maxlevel:
	1188	continue
	1189	if level > prevlevels[-1]:
	1190	# can only advance one level at a time
	1191	self.write(' '*prevlevels[-1], '<UL PLAIN>\n')
	1192	prevlevels.append(level)
	1193	elif level < prevlevels[-1]:
	1194	# might drop back multiple levels
	1195	while level < prevlevels[-1]:
	1196	del prevlevels[-1]
	1197	self.write(' '*prevlevels[-1],
	1198	'</UL>\n')
	1199	self.write(' '*level, '<LI> <A HREF="',
	1200	makefile(node), '">')
	1201	self.expand(title)
	1202	self.write('</A>\n')
	1203	self.write('</UL>\n' * len(prevlevels))
	1204
	1205	# --- Page lay-out ---
	1206
	1207	# These commands are only meaningful in printed text
	1208
	1209	def do_page(self, args): pass
	1210
	1211	def do_need(self, args): pass
	1212
	1213	def bgn_group(self, args): pass
	1214	def end_group(self): pass
	1215
	1216	# --- Line lay-out ---
	1217
	1218	def do_sp(self, args):
	1219	if self.nofill:
	1220	self.write('\n')
	1221	else:
	1222	self.write('<P>\n')
	1223
	1224	def do_hline(self, args):
	1225	self.write('<HR>')
	1226
	1227	# --- Function and variable definitions ---
	1228
	1229	def bgn_deffn(self, args):
	1230	self.write('<DL>')
	1231	self.do_deffnx(args)
	1232
	1233	def end_deffn(self):
	1234	self.write('</DL>\n')
	1235
	1236	def do_deffnx(self, args):
	1237	self.write('<DT>')
	1238	words = splitwords(args, 2)
	1239	[category, name], rest = words[:2], words[2:]
	1240	self.expand('@b{%s}' % name)
	1241	for word in rest: self.expand(' ' + makevar(word))
	1242	#self.expand(' -- ' + category)
	1243	self.write('\n<DD>')
	1244	self.index('fn', name)
	1245
	1246	def bgn_defun(self, args): self.bgn_deffn('Function ' + args)
	1247	end_defun = end_deffn
	1248	def do_defunx(self, args): self.do_deffnx('Function ' + args)
	1249
	1250	def bgn_defmac(self, args): self.bgn_deffn('Macro ' + args)
	1251	end_defmac = end_deffn
	1252	def do_defmacx(self, args): self.do_deffnx('Macro ' + args)
	1253
	1254	def bgn_defspec(self, args): self.bgn_deffn('{Special Form} ' + args)
	1255	end_defspec = end_deffn
	1256	def do_defspecx(self, args): self.do_deffnx('{Special Form} ' + args)
	1257
	1258	def bgn_defvr(self, args):
	1259	self.write('<DL>')
	1260	self.do_defvrx(args)
	1261
	1262	end_defvr = end_deffn
	1263
	1264	def do_defvrx(self, args):
	1265	self.write('<DT>')
	1266	words = splitwords(args, 2)
	1267	[category, name], rest = words[:2], words[2:]
	1268	self.expand('@code{%s}' % name)
	1269	# If there are too many arguments, show them
	1270	for word in rest: self.expand(' ' + word)
	1271	#self.expand(' -- ' + category)
	1272	self.write('\n<DD>')
	1273	self.index('vr', name)
	1274
	1275	def bgn_defvar(self, args): self.bgn_defvr('Variable ' + args)
	1276	end_defvar = end_defvr
	1277	def do_defvarx(self, args): self.do_defvrx('Variable ' + args)
	1278
	1279	def bgn_defopt(self, args): self.bgn_defvr('{User Option} ' + args)
	1280	end_defopt = end_defvr
	1281	def do_defoptx(self, args): self.do_defvrx('{User Option} ' + args)
	1282
	1283	# --- Ditto for typed languages ---
	1284
	1285	def bgn_deftypefn(self, args):
	1286	self.write('<DL>')
	1287	self.do_deftypefnx(args)
	1288
	1289	end_deftypefn = end_deffn
	1290
	1291	def do_deftypefnx(self, args):
	1292	self.write('<DT>')
	1293	words = splitwords(args, 3)
	1294	[category, datatype, name], rest = words[:3], words[3:]
	1295	self.expand('@code{%s} @b{%s}' % (datatype, name))
	1296	for word in rest: self.expand(' ' + makevar(word))
	1297	#self.expand(' -- ' + category)
	1298	self.write('\n<DD>')
	1299	self.index('fn', name)
	1300
	1301
	1302	def bgn_deftypefun(self, args): self.bgn_deftypefn('Function ' + args)
	1303	end_deftypefun = end_deftypefn
	1304	def do_deftypefunx(self, args): self.do_deftypefnx('Function ' + args)
	1305
	1306	def bgn_deftypevr(self, args):
	1307	self.write('<DL>')
	1308	self.do_deftypevrx(args)
	1309
	1310	end_deftypevr = end_deftypefn
	1311
	1312	def do_deftypevrx(self, args):
	1313	self.write('<DT>')
	1314	words = splitwords(args, 3)
	1315	[category, datatype, name], rest = words[:3], words[3:]
	1316	self.expand('@code{%s} @b{%s}' % (datatype, name))
	1317	# If there are too many arguments, show them
	1318	for word in rest: self.expand(' ' + word)
	1319	#self.expand(' -- ' + category)
	1320	self.write('\n<DD>')
	1321	self.index('fn', name)
	1322
	1323	def bgn_deftypevar(self, args):
	1324	self.bgn_deftypevr('Variable ' + args)
	1325	end_deftypevar = end_deftypevr
	1326	def do_deftypevarx(self, args):
	1327	self.do_deftypevrx('Variable ' + args)
	1328
	1329	# --- Ditto for object-oriented languages ---
	1330
	1331	def bgn_defcv(self, args):
	1332	self.write('<DL>')
	1333	self.do_defcvx(args)
	1334
	1335	end_defcv = end_deftypevr
	1336
	1337	def do_defcvx(self, args):
	1338	self.write('<DT>')
	1339	words = splitwords(args, 3)
	1340	[category, classname, name], rest = words[:3], words[3:]
	1341	self.expand('@b{%s}' % name)
	1342	# If there are too many arguments, show them
	1343	for word in rest: self.expand(' ' + word)
	1344	#self.expand(' -- %s of @code{%s}' % (category, classname))
	1345	self.write('\n<DD>')
	1346	self.index('vr', '%s @r{on %s}' % (name, classname))
	1347
	1348	def bgn_defivar(self, args):
	1349	self.bgn_defcv('{Instance Variable} ' + args)
	1350	end_defivar = end_defcv
	1351	def do_defivarx(self, args):
	1352	self.do_defcvx('{Instance Variable} ' + args)
	1353
	1354	def bgn_defop(self, args):
	1355	self.write('<DL>')
	1356	self.do_defopx(args)
	1357
	1358	end_defop = end_defcv
	1359
	1360	def do_defopx(self, args):
	1361	self.write('<DT>')
	1362	words = splitwords(args, 3)
	1363	[category, classname, name], rest = words[:3], words[3:]
	1364	self.expand('@b{%s}' % name)
	1365	for word in rest: self.expand(' ' + makevar(word))
	1366	#self.expand(' -- %s of @code{%s}' % (category, classname))
	1367	self.write('\n<DD>')
	1368	self.index('fn', '%s @r{on %s}' % (name, classname))
	1369
	1370	def bgn_defmethod(self, args):
	1371	self.bgn_defop('Method ' + args)
	1372	end_defmethod = end_defop
	1373	def do_defmethodx(self, args):
	1374	self.do_defopx('Method ' + args)
	1375
	1376	# --- Ditto for data types ---
	1377
	1378	def bgn_deftp(self, args):
	1379	self.write('<DL>')
	1380	self.do_deftpx(args)
	1381
	1382	end_deftp = end_defcv
	1383
	1384	def do_deftpx(self, args):
	1385	self.write('<DT>')
	1386	words = splitwords(args, 2)
	1387	[category, name], rest = words[:2], words[2:]
	1388	self.expand('@b{%s}' % name)
	1389	for word in rest: self.expand(' ' + word)
	1390	#self.expand(' -- ' + category)
	1391	self.write('\n<DD>')
	1392	self.index('tp', name)
	1393
	1394	# --- Making Lists and Tables
	1395
	1396	def bgn_enumerate(self, args):
	1397	if not args:
	1398	self.write('<OL>\n')
	1399	self.stackinfo[len(self.stack)] = '</OL>\n'
	1400	else:
	1401	self.itemnumber = args
	1402	self.write('<UL>\n')
	1403	self.stackinfo[len(self.stack)] = '</UL>\n'
	1404	def end_enumerate(self):
	1405	self.itemnumber = None
	1406	self.write(self.stackinfo[len(self.stack) + 1])
	1407	del self.stackinfo[len(self.stack) + 1]
	1408
	1409	def bgn_itemize(self, args):
	1410	self.itemarg = args
	1411	self.write('<UL>\n')
	1412	def end_itemize(self):
	1413	self.itemarg = None
	1414	self.write('</UL>\n')
	1415
	1416	def bgn_table(self, args):
	1417	self.itemarg = args
	1418	self.write('<DL>\n')
	1419	def end_table(self):
	1420	self.itemarg = None
	1421	self.write('</DL>\n')
	1422
	1423	def bgn_ftable(self, args):
	1424	self.itemindex = 'fn'
	1425	self.bgn_table(args)
	1426	def end_ftable(self):
	1427	self.itemindex = None
	1428	self.end_table()
	1429
	1430	def bgn_vtable(self, args):
	1431	self.itemindex = 'vr'
	1432	self.bgn_table(args)
	1433	def end_vtable(self):
	1434	self.itemindex = None
	1435	self.end_table()
	1436
	1437	def do_item(self, args):
	1438	if self.itemindex: self.index(self.itemindex, args)
	1439	if self.itemarg:
	1440	if self.itemarg[0] == '@' and self.itemarg[1] and \
	1441	self.itemarg[1] in string.ascii_letters:
	1442	args = self.itemarg + '{' + args + '}'
	1443	else:
	1444	# some other character, e.g. '-'
	1445	args = self.itemarg + ' ' + args
	1446	if self.itemnumber <> None:
	1447	args = self.itemnumber + '. ' + args
	1448	self.itemnumber = increment(self.itemnumber)
	1449	if self.stack and self.stack[-1] == 'table':
	1450	self.write('<DT>')
	1451	self.expand(args)
	1452	self.write('\n<DD>')
	1453	elif self.stack and self.stack[-1] == 'multitable':
	1454	self.write('<TR><TD>')
	1455	self.expand(args)
	1456	self.write('</TD>\n</TR>\n')
	1457	else:
	1458	self.write('<LI>')
	1459	self.expand(args)
	1460	self.write(' ')
	1461	do_itemx = do_item # XXX Should suppress leading blank line
	1462
	1463	# rpyron 2002-05-07 multitable support
	1464	def bgn_multitable(self, args):
	1465	self.itemarg = None # should be handled by columnfractions
	1466	self.write('<TABLE BORDER="">\n')
	1467	def end_multitable(self):
	1468	self.itemarg = None
	1469	self.write('</TABLE>\n<BR>\n')
	1470	def handle_columnfractions(self):
	1471	# It would be better to handle this, but for now it's in the way...
	1472	self.itemarg = None
	1473	def handle_tab(self):
	1474	self.write('</TD>\n <TD>')
	1475
	1476	# --- Enumerations, displays, quotations ---
	1477	# XXX Most of these should increase the indentation somehow
	1478
	1479	def bgn_quotation(self, args): self.write('<BLOCKQUOTE>')
	1480	def end_quotation(self): self.write('</BLOCKQUOTE>\n')
	1481
	1482	def bgn_example(self, args):
	1483	self.nofill = self.nofill + 1
	1484	self.write('<PRE>')
	1485	def end_example(self):
	1486	self.write('</PRE>\n')
	1487	self.nofill = self.nofill - 1
	1488
	1489	bgn_lisp = bgn_example # Synonym when contents are executable lisp code
	1490	end_lisp = end_example
	1491
	1492	bgn_smallexample = bgn_example # XXX Should use smaller font
	1493	end_smallexample = end_example
	1494
	1495	bgn_smalllisp = bgn_lisp # Ditto
	1496	end_smalllisp = end_lisp
	1497
	1498	bgn_display = bgn_example
	1499	end_display = end_example
	1500
	1501	bgn_format = bgn_display
	1502	end_format = end_display
	1503
	1504	def do_exdent(self, args): self.expand(args + '\n')
	1505	# XXX Should really mess with indentation
	1506
	1507	def bgn_flushleft(self, args):
	1508	self.nofill = self.nofill + 1
	1509	self.write('<PRE>\n')
	1510	def end_flushleft(self):
	1511	self.write('</PRE>\n')
	1512	self.nofill = self.nofill - 1
	1513
	1514	def bgn_flushright(self, args):
	1515	self.nofill = self.nofill + 1
	1516	self.write('<ADDRESS COMPACT>\n')
	1517	def end_flushright(self):
	1518	self.write('</ADDRESS>\n')
	1519	self.nofill = self.nofill - 1
	1520
	1521	def bgn_menu(self, args):
	1522	self.write('<DIR>\n')
	1523	self.write(' <STRONG><EM>Menu</EM></STRONG><P>\n')
	1524	self.htmlhelp.beginmenu()
	1525	def end_menu(self):
	1526	self.write('</DIR>\n')
	1527	self.htmlhelp.endmenu()
	1528
	1529	def bgn_cartouche(self, args): pass
	1530	def end_cartouche(self): pass
	1531
	1532	# --- Indices ---
	1533
	1534	def resetindex(self):
	1535	self.noncodeindices = ['cp']
	1536	self.indextitle = {}
	1537	self.indextitle['cp'] = 'Concept'
	1538	self.indextitle['fn'] = 'Function'
	1539	self.indextitle['ky'] = 'Keyword'
	1540	self.indextitle['pg'] = 'Program'
	1541	self.indextitle['tp'] = 'Type'
	1542	self.indextitle['vr'] = 'Variable'
	1543	#
	1544	self.whichindex = {}
	1545	for name in self.indextitle.keys():
	1546	self.whichindex[name] = []
	1547
	1548	def user_index(self, name, args):
	1549	if self.whichindex.has_key(name):
	1550	self.index(name, args)
	1551	else:
	1552	print '*** No index named', repr(name)
	1553
	1554	def do_cindex(self, args): self.index('cp', args)
	1555	def do_findex(self, args): self.index('fn', args)
	1556	def do_kindex(self, args): self.index('ky', args)
	1557	def do_pindex(self, args): self.index('pg', args)
	1558	def do_tindex(self, args): self.index('tp', args)
	1559	def do_vindex(self, args): self.index('vr', args)
	1560
	1561	def index(self, name, args):
	1562	self.whichindex[name].append((args, self.nodename))
	1563	self.htmlhelp.index(args, self.nodename)
	1564
	1565	def do_synindex(self, args):
	1566	words = args.split()
	1567	if len(words) <> 2:
	1568	print '*** bad @synindex', args
	1569	return
	1570	[old, new] = words
	1571	if not self.whichindex.has_key(old) or \
	1572	not self.whichindex.has_key(new):
	1573	print '*** bad key(s) in @synindex', args
	1574	return
	1575	if old <> new and \
	1576	self.whichindex[old] is not self.whichindex[new]:
	1577	inew = self.whichindex[new]
	1578	inew[len(inew):] = self.whichindex[old]
	1579	self.whichindex[old] = inew
	1580	do_syncodeindex = do_synindex # XXX Should use code font
	1581
	1582	def do_printindex(self, args):
	1583	words = args.split()
	1584	for name in words:
	1585	if self.whichindex.has_key(name):
	1586	self.prindex(name)
	1587	else:
	1588	print '*** No index named', repr(name)
	1589
	1590	def prindex(self, name):
	1591	iscodeindex = (name not in self.noncodeindices)
	1592	index = self.whichindex[name]
	1593	if not index: return
	1594	if self.debugging:
	1595	print '!'*self.debugging, '--- Generating', \
	1596	self.indextitle[name], 'index'
	1597	# The node already provides a title
	1598	index1 = []
	1599	junkprog = re.compile('^(@[a-z]+)?{')
	1600	for key, node in index:
	1601	sortkey = key.lower()
	1602	# Remove leading `@cmd{' from sort key
	1603	# -- don't bother about the matching `}'
	1604	oldsortkey = sortkey
	1605	while 1:
	1606	mo = junkprog.match(sortkey)
	1607	if not mo:
	1608	break
	1609	i = mo.end()
	1610	sortkey = sortkey[i:]
	1611	index1.append((sortkey, key, node))
	1612	del index[:]
	1613	index1.sort()
	1614	self.write('<DL COMPACT>\n')
	1615	prevkey = prevnode = None
	1616	for sortkey, key, node in index1:
	1617	if (key, node) == (prevkey, prevnode):
	1618	continue
	1619	if self.debugging > 1: print '!'*self.debugging, key, ':', node
	1620	self.write('<DT>')
	1621	if iscodeindex: key = '@code{' + key + '}'
	1622	if key != prevkey:
	1623	self.expand(key)
	1624	self.write('\n<DD><A HREF="%s">%s</A>\n' % (makefile(node), node))
	1625	prevkey, prevnode = key, node
	1626	self.write('</DL>\n')
	1627
	1628	# --- Final error reports ---
	1629
	1630	def report(self):
	1631	if self.unknown:
	1632	print '--- Unrecognized commands ---'
	1633	cmds = self.unknown.keys()
	1634	cmds.sort()
	1635	for cmd in cmds:
	1636	print cmd.ljust(20), self.unknown[cmd]
	1637
	1638
	1639	class TexinfoParserHTML3(TexinfoParser):
	1640
	1641	COPYRIGHT_SYMBOL = "©"
	1642	FN_ID_PATTERN = "[%(id)s]"
	1643	FN_SOURCE_PATTERN = '<A ID=footnoteref%(id)s ' \
	1644	'HREF="#footnotetext%(id)s">' + FN_ID_PATTERN + '</A>'
	1645	FN_TARGET_PATTERN = '<FN ID=footnotetext%(id)s>\n' \
	1646	'<P><A HREF="#footnoteref%(id)s">' + FN_ID_PATTERN \
	1647	+ '</A>\n%(text)s</P></FN>\n'
	1648	FN_HEADER = '<DIV CLASS=footnotes>\n <HR NOSHADE WIDTH=200>\n' \
	1649	' <STRONG><EM>Footnotes</EM></STRONG>\n <P>\n'
	1650
	1651	Node = HTML3Node
	1652
	1653	def bgn_quotation(self, args): self.write('<BQ>')
	1654	def end_quotation(self): self.write('</BQ>\n')
	1655
	1656	def bgn_example(self, args):
	1657	# this use of <CODE> would not be legal in HTML 2.0,
	1658	# but is in more recent DTDs.
	1659	self.nofill = self.nofill + 1
	1660	self.write('<PRE CLASS=example><CODE>')
	1661	def end_example(self):
	1662	self.write("</CODE></PRE>\n")
	1663	self.nofill = self.nofill - 1
	1664
	1665	def bgn_flushleft(self, args):
	1666	self.nofill = self.nofill + 1
	1667	self.write('<PRE CLASS=flushleft>\n')
	1668
	1669	def bgn_flushright(self, args):
	1670	self.nofill = self.nofill + 1
	1671	self.write('<DIV ALIGN=right CLASS=flushright><ADDRESS COMPACT>\n')
	1672	def end_flushright(self):
	1673	self.write('</ADDRESS></DIV>\n')
	1674	self.nofill = self.nofill - 1
	1675
	1676	def bgn_menu(self, args):
	1677	self.write('<UL PLAIN CLASS=menu>\n')
	1678	self.write(' <LH>Menu</LH>\n')
	1679	def end_menu(self):
	1680	self.write('</UL>\n')
	1681
	1682
	1683	# rpyron 2002-05-07
	1684	class HTMLHelp:
	1685	"""
	1686	This class encapsulates support for HTML Help. Node names,
	1687	file names, menu items, index items, and image file names are
	1688	accumulated until a call to finalize(). At that time, three
	1689	output files are created in the current directory:
	1690
	1691	`helpbase`.hhp is a HTML Help Workshop project file.
	1692	It contains various information, some of
	1693	which I do not understand; I just copied
	1694	the default project info from a fresh
	1695	installation.
	1696	`helpbase`.hhc is the Contents file for the project.
	1697	`helpbase`.hhk is the Index file for the project.
	1698
	1699	When these files are used as input to HTML Help Workshop,
	1700	the resulting file will be named:
	1701
	1702	`helpbase`.chm
	1703
	1704	If none of the defaults in `helpbase`.hhp are changed,
	1705	the .CHM file will have Contents, Index, Search, and
	1706	Favorites tabs.
	1707	"""
	1708
	1709	codeprog = re.compile('@code{(.*?)}')
	1710
	1711	def __init__(self,helpbase,dirname):
	1712	self.helpbase = helpbase
	1713	self.dirname = dirname
	1714	self.projectfile = None
	1715	self.contentfile = None
	1716	self.indexfile = None
	1717	self.nodelist = []
	1718	self.nodenames = {} # nodename : index
	1719	self.nodeindex = {}
	1720	self.filenames = {} # filename : filename
	1721	self.indexlist = [] # (args,nodename) == (key,location)
	1722	self.current = ''
	1723	self.menudict = {}
	1724	self.dumped = {}
	1725
	1726
	1727	def addnode(self,name,next,prev,up,filename):
	1728	node = (name,next,prev,up,filename)
	1729	# add this file to dict
	1730	# retrieve list with self.filenames.values()
	1731	self.filenames[filename] = filename
	1732	# add this node to nodelist
	1733	self.nodeindex[name] = len(self.nodelist)
	1734	self.nodelist.append(node)
	1735	# set 'current' for menu items
	1736	self.current = name
	1737	self.menudict[self.current] = []
	1738
	1739	def menuitem(self,nodename):
	1740	menu = self.menudict[self.current]
	1741	menu.append(nodename)
	1742
	1743
	1744	def addimage(self,imagename):
	1745	self.filenames[imagename] = imagename
	1746
	1747	def index(self, args, nodename):
	1748	self.indexlist.append((args,nodename))
	1749
	1750	def beginmenu(self):
	1751	pass
	1752
	1753	def endmenu(self):
	1754	pass
	1755
	1756	def finalize(self):
	1757	if not self.helpbase:
	1758	return
	1759
	1760	# generate interesting filenames
	1761	resultfile = self.helpbase + '.chm'
	1762	projectfile = self.helpbase + '.hhp'
	1763	contentfile = self.helpbase + '.hhc'
	1764	indexfile = self.helpbase + '.hhk'
	1765
	1766	# generate a reasonable title
	1767	title = self.helpbase
	1768
	1769	# get the default topic file
	1770	(topname,topnext,topprev,topup,topfile) = self.nodelist[0]
	1771	defaulttopic = topfile
	1772
	1773	# PROJECT FILE
	1774	try:
	1775	fp = open(projectfile,'w')
	1776	print>>fp, '[OPTIONS]'
	1777	print>>fp, 'Auto Index=Yes'
	1778	print>>fp, 'Binary TOC=No'
	1779	print>>fp, 'Binary Index=Yes'
	1780	print>>fp, 'Compatibility=1.1'
	1781	print>>fp, 'Compiled file=' + resultfile + ''
	1782	print>>fp, 'Contents file=' + contentfile + ''
	1783	print>>fp, 'Default topic=' + defaulttopic + ''
	1784	print>>fp, 'Error log file=ErrorLog.log'
	1785	print>>fp, 'Index file=' + indexfile + ''
	1786	print>>fp, 'Title=' + title + ''
	1787	print>>fp, 'Display compile progress=Yes'
	1788	print>>fp, 'Full-text search=Yes'
	1789	print>>fp, 'Default window=main'
	1790	print>>fp, ''
	1791	print>>fp, '[WINDOWS]'
	1792	print>>fp, ('main=,"' + contentfile + '","' + indexfile
	1793	+ '","","",,,,,0x23520,222,0x1046,[10,10,780,560],'
	1794	'0xB0000,,,,,,0')
	1795	print>>fp, ''
	1796	print>>fp, '[FILES]'
	1797	print>>fp, ''
	1798	self.dumpfiles(fp)
	1799	fp.close()
	1800	except IOError, msg:
	1801	print projectfile, ':', msg
	1802	sys.exit(1)
	1803
	1804	# CONTENT FILE
	1805	try:
	1806	fp = open(contentfile,'w')
	1807	print>>fp, '<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML//EN">'
	1808	print>>fp, '<!-- This file defines the table of contents -->'
	1809	print>>fp, '<HTML>'
	1810	print>>fp, '<HEAD>'
	1811	print>>fp, ('<meta name="GENERATOR"'
	1812	'content="Microsoft® HTML Help Workshop 4.1">')
	1813	print>>fp, '<!-- Sitemap 1.0 -->'
	1814	print>>fp, '</HEAD>'
	1815	print>>fp, '<BODY>'
	1816	print>>fp, ' <OBJECT type="text/site properties">'
	1817	print>>fp, ' <param name="Window Styles" value="0x800025">'
	1818	print>>fp, ' <param name="comment" value="title:">'
	1819	print>>fp, ' <param name="comment" value="base:">'
	1820	print>>fp, ' </OBJECT>'
	1821	self.dumpnodes(fp)
	1822	print>>fp, '</BODY>'
	1823	print>>fp, '</HTML>'
	1824	fp.close()
	1825	except IOError, msg:
	1826	print contentfile, ':', msg
	1827	sys.exit(1)
	1828
	1829	# INDEX FILE
	1830	try:
	1831	fp = open(indexfile ,'w')
	1832	print>>fp, '<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML//EN">'
	1833	print>>fp, '<!-- This file defines the index -->'
	1834	print>>fp, '<HTML>'
	1835	print>>fp, '<HEAD>'
	1836	print>>fp, ('<meta name="GENERATOR"'
	1837	'content="Microsoft® HTML Help Workshop 4.1">')
	1838	print>>fp, '<!-- Sitemap 1.0 -->'
	1839	print>>fp, '</HEAD>'
	1840	print>>fp, '<BODY>'
	1841	print>>fp, '<OBJECT type="text/site properties">'
	1842	print>>fp, '</OBJECT>'
	1843	self.dumpindex(fp)
	1844	print>>fp, '</BODY>'
	1845	print>>fp, '</HTML>'
	1846	fp.close()
	1847	except IOError, msg:
	1848	print indexfile , ':', msg
	1849	sys.exit(1)
	1850
	1851	def dumpfiles(self, outfile=sys.stdout):
	1852	filelist = self.filenames.values()
	1853	filelist.sort()
	1854	for filename in filelist:
	1855	print>>outfile, filename
	1856
	1857	def dumpnodes(self, outfile=sys.stdout):
	1858	self.dumped = {}
	1859	if self.nodelist:
	1860	nodename, dummy, dummy, dummy, dummy = self.nodelist[0]
	1861	self.topnode = nodename
	1862
	1863	print>>outfile, '<UL>'
	1864	for node in self.nodelist:
	1865	self.dumpnode(node,0,outfile)
	1866	print>>outfile, '</UL>'
	1867
	1868	def dumpnode(self, node, indent=0, outfile=sys.stdout):
	1869	if node:
	1870	# Retrieve info for this node
	1871	(nodename,next,prev,up,filename) = node
	1872	self.current = nodename
	1873
	1874	# Have we been dumped already?
	1875	if self.dumped.has_key(nodename):
	1876	return
	1877	self.dumped[nodename] = 1
	1878
	1879	# Print info for this node
	1880	print>>outfile, ' '*indent,
	1881	print>>outfile, '<LI><OBJECT type="text/sitemap">',
	1882	print>>outfile, '<param name="Name" value="' + nodename +'">',
	1883	print>>outfile, '<param name="Local" value="'+ filename +'">',
	1884	print>>outfile, '</OBJECT>'
	1885
	1886	# Does this node have menu items?
	1887	try:
	1888	menu = self.menudict[nodename]
	1889	self.dumpmenu(menu,indent+2,outfile)
	1890	except KeyError:
	1891	pass
	1892
	1893	def dumpmenu(self, menu, indent=0, outfile=sys.stdout):
	1894	if menu:
	1895	currentnode = self.current
	1896	if currentnode != self.topnode: # XXX this is a hack
	1897	print>>outfile, ' '*indent + '<UL>'
	1898	indent += 2
	1899	for item in menu:
	1900	menunode = self.getnode(item)
	1901	self.dumpnode(menunode,indent,outfile)
	1902	if currentnode != self.topnode: # XXX this is a hack
	1903	print>>outfile, ' '*indent + '</UL>'
	1904	indent -= 2
	1905
	1906	def getnode(self, nodename):
	1907	try:
	1908	index = self.nodeindex[nodename]
	1909	return self.nodelist[index]
	1910	except KeyError:
	1911	return None
	1912	except IndexError:
	1913	return None
	1914
	1915	# (args,nodename) == (key,location)
	1916	def dumpindex(self, outfile=sys.stdout):
	1917	print>>outfile, '<UL>'
	1918	for (key,location) in self.indexlist:
	1919	key = self.codeexpand(key)
	1920	location = makefile(location)
	1921	location = self.dirname + '/' + location
	1922	print>>outfile, '<LI><OBJECT type="text/sitemap">',
	1923	print>>outfile, '<param name="Name" value="' + key + '">',
	1924	print>>outfile, '<param name="Local" value="' + location + '">',
	1925	print>>outfile, '</OBJECT>'
	1926	print>>outfile, '</UL>'
	1927
	1928	def codeexpand(self, line):
	1929	co = self.codeprog.match(line)
	1930	if not co:
	1931	return line
	1932	bgn, end = co.span(0)
	1933	a, b = co.span(1)
	1934	line = line[:bgn] + line[a:b] + line[end:]
	1935	return line
	1936
	1937
	1938	# Put @var{} around alphabetic substrings
	1939	def makevar(str):
	1940	return '@var{'+str+'}'
	1941
	1942
	1943	# Split a string in "words" according to findwordend
	1944	def splitwords(str, minlength):
	1945	words = []
	1946	i = 0
	1947	n = len(str)
	1948	while i < n:
	1949	while i < n and str[i] in ' \t\n': i = i+1
	1950	if i >= n: break
	1951	start = i
	1952	i = findwordend(str, i, n)
	1953	words.append(str[start:i])
	1954	while len(words) < minlength: words.append('')
	1955	return words
	1956
	1957
	1958	# Find the end of a "word", matching braces and interpreting @@ @{ @}
	1959	fwprog = re.compile('[@{} ]')
	1960	def findwordend(str, i, n):
	1961	level = 0
	1962	while i < n:
	1963	mo = fwprog.search(str, i)
	1964	if not mo:
	1965	break
	1966	i = mo.start()
	1967	c = str[i]; i = i+1
	1968	if c == '@': i = i+1 # Next character is not special
	1969	elif c == '{': level = level+1
	1970	elif c == '}': level = level-1
	1971	elif c == ' ' and level <= 0: return i-1
	1972	return n
	1973
	1974
	1975	# Convert a node name into a file name
	1976	def makefile(nodename):
	1977	nodename = nodename.strip()
	1978	return fixfunnychars(nodename) + '.html'
	1979
	1980
	1981	# Characters that are perfectly safe in filenames and hyperlinks
	1982	goodchars = string.ascii_letters + string.digits + '!@-=+.'
	1983
	1984	# Replace characters that aren't perfectly safe by dashes
	1985	# Underscores are bad since Cern HTTPD treats them as delimiters for
	1986	# encoding times, so you get mismatches if you compress your files:
	1987	# a.html.gz will map to a_b.html.gz
	1988	def fixfunnychars(addr):
	1989	i = 0
	1990	while i < len(addr):
	1991	c = addr[i]
	1992	if c not in goodchars:
	1993	c = '-'
	1994	addr = addr[:i] + c + addr[i+1:]
	1995	i = i + len(c)
	1996	return addr
	1997
	1998
	1999	# Increment a string used as an enumeration
	2000	def increment(s):
	2001	if not s:
	2002	return '1'
	2003	for sequence in string.digits, string.lowercase, string.uppercase:
	2004	lastc = s[-1]
	2005	if lastc in sequence:
	2006	i = sequence.index(lastc) + 1
	2007	if i >= len(sequence):
	2008	if len(s) == 1:
	2009	s = sequence[0]*2
	2010	if s == '00':
	2011	s = '10'
	2012	else:
	2013	s = increment(s[:-1]) + sequence[0]
	2014	else:
	2015	s = s[:-1] + sequence[i]
	2016	return s
	2017	return s # Don't increment
	2018
	2019
	2020	def test():
	2021	import sys
	2022	debugging = 0
	2023	print_headers = 0
	2024	cont = 0
	2025	html3 = 0
	2026	htmlhelp = ''
	2027
	2028	while sys.argv[1] == ['-d']:
	2029	debugging = debugging + 1
	2030	del sys.argv[1]
	2031	if sys.argv[1] == '-p':
	2032	print_headers = 1
	2033	del sys.argv[1]
	2034	if sys.argv[1] == '-c':
	2035	cont = 1
	2036	del sys.argv[1]
	2037	if sys.argv[1] == '-3':
	2038	html3 = 1
	2039	del sys.argv[1]
	2040	if sys.argv[1] == '-H':
	2041	helpbase = sys.argv[2]
	2042	del sys.argv[1:3]
	2043	if len(sys.argv) <> 3:
	2044	print 'usage: texi2hh [-d [-d]] [-p] [-c] [-3] [-H htmlhelp]', \
	2045	'inputfile outputdirectory'
	2046	sys.exit(2)
	2047
	2048	if html3:
	2049	parser = TexinfoParserHTML3()
	2050	else:
	2051	parser = TexinfoParser()
	2052	parser.cont = cont
	2053	parser.debugging = debugging
	2054	parser.print_headers = print_headers
	2055
	2056	file = sys.argv[1]
	2057	dirname = sys.argv[2]
	2058	parser.setdirname(dirname)
	2059	parser.setincludedir(os.path.dirname(file))
	2060
	2061	htmlhelp = HTMLHelp(helpbase, dirname)
	2062	parser.sethtmlhelp(htmlhelp)
	2063
	2064	try:
	2065	fp = open(file, 'r')
	2066	except IOError, msg:
	2067	print file, ':', msg
	2068	sys.exit(1)
	2069
	2070	parser.parse(fp)
	2071	fp.close()
	2072	parser.report()
	2073
	2074	htmlhelp.finalize()
	2075
	2076
	2077	if __name__ == "__main__":
	2078	test()

sys/src/cmd/python/Tools/scripts/treesync.py+205−0

@@ -0,0 +1,205 @@
	1	#! /usr/bin/env python
	2
	3	"""Script to synchronize two source trees.
	4
	5	Invoke with two arguments:
	6
	7	python treesync.py slave master
	8
	9	The assumption is that "master" contains CVS administration while
	10	slave doesn't. All files in the slave tree that have a CVS/Entries
	11	entry in the master tree are synchronized. This means:
	12
	13	If the files differ:
	14	if the slave file is newer:
	15	normalize the slave file
	16	if the files still differ:
	17	copy the slave to the master
	18	else (the master is newer):
	19	copy the master to the slave
	20
	21	normalizing the slave means replacing CRLF with LF when the master
	22	doesn't use CRLF
	23
	24	"""
	25
	26	import os, sys, stat, getopt
	27
	28	# Interactivity options
	29	default_answer = "ask"
	30	create_files = "yes"
	31	create_directories = "no"
	32	write_slave = "ask"
	33	write_master = "ask"
	34
	35	def main():
	36	global always_no, always_yes
	37	global create_directories, write_master, write_slave
	38	opts, args = getopt.getopt(sys.argv[1:], "nym:s:d:f:a:")
	39	for o, a in opts:
	40	if o == '-y':
	41	default_answer = "yes"
	42	if o == '-n':
	43	default_answer = "no"
	44	if o == '-s':
	45	write_slave = a
	46	if o == '-m':
	47	write_master = a
	48	if o == '-d':
	49	create_directories = a
	50	if o == '-f':
	51	create_files = a
	52	if o == '-a':
	53	create_files = create_directories = write_slave = write_master = a
	54	try:
	55	[slave, master] = args
	56	except ValueError:
	57	print "usage: python", sys.argv[0] or "treesync.py",
	58	print "[-n] [-y] [-m y\|n\|a] [-s y\|n\|a] [-d y\|n\|a] [-f n\|y\|a]",
	59	print "slavedir masterdir"
	60	return
	61	process(slave, master)
	62
	63	def process(slave, master):
	64	cvsdir = os.path.join(master, "CVS")
	65	if not os.path.isdir(cvsdir):
	66	print "skipping master subdirectory", master
	67	print "-- not under CVS"
	68	return
	69	print "-"*40
	70	print "slave ", slave
	71	print "master", master
	72	if not os.path.isdir(slave):
	73	if not okay("create slave directory %s?" % slave,
	74	answer=create_directories):
	75	print "skipping master subdirectory", master
	76	print "-- no corresponding slave", slave
	77	return
	78	print "creating slave directory", slave
	79	try:
	80	os.mkdir(slave)
	81	except os.error, msg:
	82	print "can't make slave directory", slave, ":", msg
	83	return
	84	else:
	85	print "made slave directory", slave
	86	cvsdir = None
	87	subdirs = []
	88	names = os.listdir(master)
	89	for name in names:
	90	mastername = os.path.join(master, name)
	91	slavename = os.path.join(slave, name)
	92	if name == "CVS":
	93	cvsdir = mastername
	94	else:
	95	if os.path.isdir(mastername) and not os.path.islink(mastername):
	96	subdirs.append((slavename, mastername))
	97	if cvsdir:
	98	entries = os.path.join(cvsdir, "Entries")
	99	for e in open(entries).readlines():
	100	words = e.split('/')
	101	if words[0] == '' and words[1:]:
	102	name = words[1]
	103	s = os.path.join(slave, name)
	104	m = os.path.join(master, name)
	105	compare(s, m)
	106	for (s, m) in subdirs:
	107	process(s, m)
	108
	109	def compare(slave, master):
	110	try:
	111	sf = open(slave, 'r')
	112	except IOError:
	113	sf = None
	114	try:
	115	mf = open(master, 'rb')
	116	except IOError:
	117	mf = None
	118	if not sf:
	119	if not mf:
	120	print "Neither master nor slave exists", master
	121	return
	122	print "Creating missing slave", slave
	123	copy(master, slave, answer=create_files)
	124	return
	125	if not mf:
	126	print "Not updating missing master", master
	127	return
	128	if sf and mf:
	129	if identical(sf, mf):
	130	return
	131	sft = mtime(sf)
	132	mft = mtime(mf)
	133	if mft > sft:
	134	# Master is newer -- copy master to slave
	135	sf.close()
	136	mf.close()
	137	print "Master ", master
	138	print "is newer than slave", slave
	139	copy(master, slave, answer=write_slave)
	140	return
	141	# Slave is newer -- copy slave to master
	142	print "Slave is", sft-mft, "seconds newer than master"
	143	# But first check what to do about CRLF
	144	mf.seek(0)
	145	fun = funnychars(mf)
	146	mf.close()
	147	sf.close()
	148	if fun:
	149	print "*UPDATING MASTER (BINARY COPY)*"
	150	copy(slave, master, "rb", answer=write_master)
	151	else:
	152	print "*UPDATING MASTER*"
	153	copy(slave, master, "r", answer=write_master)
	154
	155	BUFSIZE = 16*1024
	156
	157	def identical(sf, mf):
	158	while 1:
	159	sd = sf.read(BUFSIZE)
	160	md = mf.read(BUFSIZE)
	161	if sd != md: return 0
	162	if not sd: break
	163	return 1
	164
	165	def mtime(f):
	166	st = os.fstat(f.fileno())
	167	return st[stat.ST_MTIME]
	168
	169	def funnychars(f):
	170	while 1:
	171	buf = f.read(BUFSIZE)
	172	if not buf: break
	173	if '\r' in buf or '\0' in buf: return 1
	174	return 0
	175
	176	def copy(src, dst, rmode="rb", wmode="wb", answer='ask'):
	177	print "copying", src
	178	print " to", dst
	179	if not okay("okay to copy? ", answer):
	180	return
	181	f = open(src, rmode)
	182	g = open(dst, wmode)
	183	while 1:
	184	buf = f.read(BUFSIZE)
	185	if not buf: break
	186	g.write(buf)
	187	f.close()
	188	g.close()
	189
	190	def okay(prompt, answer='ask'):
	191	answer = answer.strip().lower()
	192	if not answer or answer[0] not in 'ny':
	193	answer = raw_input(prompt)
	194	answer = answer.strip().lower()
	195	if not answer:
	196	answer = default_answer
	197	if answer[:1] == 'y':
	198	return 1
	199	if answer[:1] == 'n':
	200	return 0
	201	print "Yes or No please -- try again:"
	202	return okay(prompt)
	203
	204	if __name__ == '__main__':
	205	main()

sys/src/cmd/python/Tools/scripts/untabify.py+52−0

@@ -0,0 +1,52 @@
	1	#! /usr/bin/env python
	2
	3	"Replace tabs with spaces in argument files. Print names of changed files."
	4
	5	import os
	6	import sys
	7	import getopt
	8
	9	def main():
	10	tabsize = 8
	11	try:
	12	opts, args = getopt.getopt(sys.argv[1:], "t:")
	13	if not args:
	14	raise getopt.error, "At least one file argument required"
	15	except getopt.error, msg:
	16	print msg
	17	print "usage:", sys.argv[0], "[-t tabwidth] file ..."
	18	return
	19	for optname, optvalue in opts:
	20	if optname == '-t':
	21	tabsize = int(optvalue)
	22
	23	for filename in args:
	24	process(filename, tabsize)
	25
	26	def process(filename, tabsize):
	27	try:
	28	f = open(filename)
	29	text = f.read()
	30	f.close()
	31	except IOError, msg:
	32	print "%r: I/O error: %s" % (filename, msg)
	33	return
	34	newtext = text.expandtabs(tabsize)
	35	if newtext == text:
	36	return
	37	backup = filename + "~"
	38	try:
	39	os.unlink(backup)
	40	except os.error:
	41	pass
	42	try:
	43	os.rename(filename, backup)
	44	except os.error:
	45	pass
	46	f = open(filename, "w")
	47	f.write(newtext)
	48	f.close()
	49	print filename
	50
	51	if __name__ == '__main__':
	52	main()

sys/src/cmd/python/Tools/scripts/which.py+60−0

@@ -0,0 +1,60 @@
	1	#! /usr/bin/env python
	2
	3	# Variant of "which".
	4	# On stderr, near and total misses are reported.
	5	# '-l<flags>' argument adds ls -l<flags> of each file found.
	6
	7	import sys
	8	if sys.path[0] in (".", ""): del sys.path[0]
	9
	10	import sys, os
	11	from stat import *
	12
	13	def msg(str):
	14	sys.stderr.write(str + '\n')
	15
	16	def main():
	17	pathlist = os.environ['PATH'].split(os.pathsep)
	18
	19	sts = 0
	20	longlist = ''
	21
	22	if sys.argv[1:] and sys.argv[1][:2] == '-l':
	23	longlist = sys.argv[1]
	24	del sys.argv[1]
	25
	26	for prog in sys.argv[1:]:
	27	ident = ()
	28	for dir in pathlist:
	29	filename = os.path.join(dir, prog)
	30	try:
	31	st = os.stat(filename)
	32	except os.error:
	33	continue
	34	if not S_ISREG(st[ST_MODE]):
	35	msg(filename + ': not a disk file')
	36	else:
	37	mode = S_IMODE(st[ST_MODE])
	38	if mode & 0111:
	39	if not ident:
	40	print filename
	41	ident = st[:3]
	42	else:
	43	if st[:3] == ident:
	44	s = 'same as: '
	45	else:
	46	s = 'also: '
	47	msg(s + filename)
	48	else:
	49	msg(filename + ': not executable')
	50	if longlist:
	51	sts = os.system('ls ' + longlist + ' ' + filename)
	52	if sts: msg('"ls -l" exit status: ' + repr(sts))
	53	if not ident:
	54	msg(prog + ': not found')
	55	sts = 1
	56
	57	sys.exit(sts)
	58
	59	if __name__ == '__main__':
	60	main()

sys/src/cmd/python/Tools/scripts/xxci.py+117−0

@@ -0,0 +1,117 @@
	1	#! /usr/bin/env python
	2
	3	# xxci
	4	#
	5	# check in files for which rcsdiff returns nonzero exit status
	6
	7	import sys
	8	import os
	9	from stat import *
	10	import commands
	11	import fnmatch
	12
	13	EXECMAGIC = '\001\140\000\010'
	14
	15	MAXSIZE = 200*1024 # Files this big must be binaries and are skipped.
	16
	17	def getargs():
	18	args = sys.argv[1:]
	19	if args:
	20	return args
	21	print 'No arguments, checking almost *, in "ls -t" order'
	22	list = []
	23	for file in os.listdir(os.curdir):
	24	if not skipfile(file):
	25	list.append((getmtime(file), file))
	26	list.sort()
	27	if not list:
	28	print 'Nothing to do -- exit 1'
	29	sys.exit(1)
	30	list.sort()
	31	list.reverse()
	32	for mtime, file in list: args.append(file)
	33	return args
	34
	35	def getmtime(file):
	36	try:
	37	st = os.stat(file)
	38	return st[ST_MTIME]
	39	except os.error:
	40	return -1
	41
	42	badnames = ['tags', 'TAGS', 'xyzzy', 'nohup.out', 'core']
	43	badprefixes = ['.', ',', '@', '#', 'o.']
	44	badsuffixes = \
	45	['~', '.a', '.o', '.old', '.bak', '.orig', '.new', '.prev', '.not', \
	46	'.pyc', '.fdc', '.rgb', '.elc', ',v']
	47	ignore = []
	48
	49	def setup():
	50	ignore[:] = badnames
	51	for p in badprefixes:
	52	ignore.append(p + '*')
	53	for p in badsuffixes:
	54	ignore.append('*' + p)
	55	try:
	56	f = open('.xxcign', 'r')
	57	except IOError:
	58	return
	59	ignore[:] = ignore + f.read().split()
	60
	61	def skipfile(file):
	62	for p in ignore:
	63	if fnmatch.fnmatch(file, p): return 1
	64	try:
	65	st = os.lstat(file)
	66	except os.error:
	67	return 1 # Doesn't exist -- skip it
	68	# Skip non-plain files.
	69	if not S_ISREG(st[ST_MODE]): return 1
	70	# Skip huge files -- probably binaries.
	71	if st[ST_SIZE] >= MAXSIZE: return 1
	72	# Skip executables
	73	try:
	74	data = open(file, 'r').read(len(EXECMAGIC))
	75	if data == EXECMAGIC: return 1
	76	except:
	77	pass
	78	return 0
	79
	80	def badprefix(file):
	81	for bad in badprefixes:
	82	if file[:len(bad)] == bad: return 1
	83	return 0
	84
	85	def badsuffix(file):
	86	for bad in badsuffixes:
	87	if file[-len(bad):] == bad: return 1
	88	return 0
	89
	90	def go(args):
	91	for file in args:
	92	print file + ':'
	93	if differing(file):
	94	showdiffs(file)
	95	if askyesno('Check in ' + file + ' ? '):
	96	sts = os.system('rcs -l ' + file) # ignored
	97	sts = os.system('ci -l ' + file)
	98
	99	def differing(file):
	100	cmd = 'co -p ' + file + ' 2>/dev/null \| cmp -s - ' + file
	101	sts = os.system(cmd)
	102	return sts != 0
	103
	104	def showdiffs(file):
	105	cmd = 'rcsdiff ' + file + ' 2>&1 \| ${PAGER-more}'
	106	sts = os.system(cmd)
	107
	108	def askyesno(prompt):
	109	s = raw_input(prompt)
	110	return s in ['y', 'yes']
	111
	112	if __name__ == '__main__':
	113	try:
	114	setup()
	115	go(getargs())
	116	except KeyboardInterrupt:
	117	print '[Intr]'