add hg and python

cinap_lenrek committed 15 years ago

Commit 4bb9e221 parent 987265c3706 files · +1228772 −-233Browse files

Showing the changes to sys/src/cmd/python/Tools/unicode/mkstringprep.py (1 of 3706 files). Show all

Unified · Split

sys/src/cmd/python/Tools/unicode/mkstringprep.py+431−0

@@ -0,0 +1,431 @@
	1	+import re, unicodedata, sys
	2	+
	3	+if sys.maxunicode == 65535:
	4	+ raise RuntimeError, "need UCS-4 Python"
	5	+
	6	+def gen_category(cats):
	7	+ for i in range(0, 0x110000):
	8	+ if unicodedata.category(unichr(i)) in cats:
	9	+ yield(i)
	10	+
	11	+def gen_bidirectional(cats):
	12	+ for i in range(0, 0x110000):
	13	+ if unicodedata.bidirectional(unichr(i)) in cats:
	14	+ yield(i)
	15	+
	16	+def compact_set(l):
	17	+ single = []
	18	+ tuple = []
	19	+ prev = None
	20	+ span = 0
	21	+ for e in l:
	22	+ if prev is None:
	23	+ prev = e
	24	+ span = 0
	25	+ continue
	26	+ if prev+span+1 != e:
	27	+ if span > 2:
	28	+ tuple.append((prev,prev+span+1))
	29	+ else:
	30	+ for i in range(prev, prev+span+1):
	31	+ single.append(i)
	32	+ prev = e
	33	+ span = 0
	34	+ else:
	35	+ span += 1
	36	+ if span:
	37	+ tuple.append((prev,prev+span+1))
	38	+ else:
	39	+ single.append(prev)
	40	+ tuple = " + ".join(["range(%d,%d)" % t for t in tuple])
	41	+ if not single:
	42	+ return "set(%s)" % tuple
	43	+ if not tuple:
	44	+ return "set(%s)" % repr(single)
	45	+ return "set(%s + %s)" % (repr(single),tuple)
	46	+
	47	+############## Read the tables in the RFC #######################
	48	+
	49	+data = open("rfc3454.txt").readlines()
	50	+
	51	+tables = []
	52	+curname = None
	53	+for l in data:
	54	+ l = l.strip()
	55	+ if not l:
	56	+ continue
	57	+ # Skip RFC page breaks
	58	+ if l.startswith("Hoffman & Blanchet") or\
	59	+ l.startswith("RFC 3454"):
	60	+ continue
	61	+ # Find start/end lines
	62	+ m = re.match("----- (Start\|End) Table ([A-Z](.[0-9])+) -----", l)
	63	+ if m:
	64	+ if m.group(1) == "Start":
	65	+ if curname:
	66	+ raise "Double Start",(curname, l)
	67	+ curname = m.group(2)
	68	+ table = {}
	69	+ tables.append((curname, table))
	70	+ continue
	71	+ else:
	72	+ if not curname:
	73	+ raise "End without start", l
	74	+ curname = None
	75	+ continue
	76	+ if not curname:
	77	+ continue
	78	+ # Now we are in a table
	79	+ fields = l.split(";")
	80	+ if len(fields) > 1:
	81	+ # Drop comment field
	82	+ fields = fields[:-1]
	83	+ if len(fields) == 1:
	84	+ fields = fields[0].split("-")
	85	+ if len(fields) > 1:
	86	+ # range
	87	+ try:
	88	+ start, end = fields
	89	+ except ValueError:
	90	+ raise "Unpacking problem", l
	91	+ else:
	92	+ start = end = fields[0]
	93	+ start = int(start, 16)
	94	+ end = int(end, 16)
	95	+ for i in range(start, end+1):
	96	+ table[i] = i
	97	+ else:
	98	+ code, value = fields
	99	+ value = value.strip()
	100	+ if value:
	101	+ value = [int(v, 16) for v in value.split(" ")]
	102	+ else:
	103	+ # table B.1
	104	+ value = None
	105	+ table[int(code, 16)] = value
	106	+
	107	+########### Generate compact Python versions of the tables #############
	108	+
	109	+print """# This file is generated by mkstringprep.py. DO NOT EDIT.
	110	+\"\"\"Library that exposes various tables found in the StringPrep RFC 3454.
	111	+
	112	+There are two kinds of tables: sets, for which a member test is provided,
	113	+and mappings, for which a mapping function is provided.
	114	+\"\"\"
	115	+
	116	+import unicodedata
	117	+"""
	118	+
	119	+print "assert unicodedata.unidata_version == %s" % repr(unicodedata.unidata_version)
	120	+
	121	+# A.1 is the table of unassigned characters
	122	+# XXX Plane 15 PUA is listed as unassigned in Python.
	123	+name, table = tables[0]
	124	+del tables[0]
	125	+assert name == "A.1"
	126	+table = set(table.keys())
	127	+Cn = set(gen_category(["Cn"]))
	128	+
	129	+# FDD0..FDEF are process internal codes
	130	+Cn -= set(range(0xFDD0, 0xFDF0))
	131	+# not a character
	132	+Cn -= set(range(0xFFFE, 0x110000, 0x10000))
	133	+Cn -= set(range(0xFFFF, 0x110000, 0x10000))
	134	+
	135	+# assert table == Cn
	136	+
	137	+print """
	138	+def in_table_a1(code):
	139	+ if unicodedata.category(code) != 'Cn': return False
	140	+ c = ord(code)
	141	+ if 0xFDD0 <= c < 0xFDF0: return False
	142	+ return (c & 0xFFFF) not in (0xFFFE, 0xFFFF)
	143	+"""
	144	+
	145	+# B.1 cannot easily be derived
	146	+name, table = tables[0]
	147	+del tables[0]
	148	+assert name == "B.1"
	149	+table = table.keys()
	150	+table.sort()
	151	+print """
	152	+b1_set = """ + compact_set(table) + """
	153	+def in_table_b1(code):
	154	+ return ord(code) in b1_set
	155	+"""
	156	+
	157	+# B.2 and B.3 is case folding.
	158	+# It takes CaseFolding.txt into account, which is
	159	+# not available in the Python database. Since
	160	+# B.2 is derived from B.3, we process B.3 first.
	161	+# B.3 supposedly is CaseFolding-3.2.0.txt.
	162	+
	163	+name, table_b2 = tables[0]
	164	+del tables[0]
	165	+assert name == "B.2"
	166	+
	167	+name, table_b3 = tables[0]
	168	+del tables[0]
	169	+assert name == "B.3"
	170	+
	171	+# B.3 is mostly Python's .lower, except for a number
	172	+# of special cases, e.g. considering canonical forms.
	173	+
	174	+b3_exceptions = {}
	175	+
	176	+for k,v in table_b2.items():
	177	+ if map(ord, unichr(k).lower()) != v:
	178	+ b3_exceptions[k] = u"".join(map(unichr,v))
	179	+
	180	+b3 = b3_exceptions.items()
	181	+b3.sort()
	182	+
	183	+print """
	184	+b3_exceptions = {"""
	185	+for i,(k,v) in enumerate(b3):
	186	+ print "0x%x:%s," % (k, repr(v)),
	187	+ if i % 4 == 3:
	188	+ print
	189	+print "}"
	190	+
	191	+print """
	192	+def map_table_b3(code):
	193	+ r = b3_exceptions.get(ord(code))
	194	+ if r is not None: return r
	195	+ return code.lower()
	196	+"""
	197	+
	198	+def map_table_b3(code):
	199	+ r = b3_exceptions.get(ord(code))
	200	+ if r is not None: return r
	201	+ return code.lower()
	202	+
	203	+# B.2 is case folding for NFKC. This is the same as B.3,
	204	+# except where NormalizeWithKC(Fold(a)) !=
	205	+# NormalizeWithKC(Fold(NormalizeWithKC(Fold(a))))
	206	+
	207	+def map_table_b2(a):
	208	+ al = map_table_b3(a)
	209	+ b = unicodedata.normalize("NFKC", al)
	210	+ bl = u"".join([map_table_b3(ch) for ch in b])
	211	+ c = unicodedata.normalize("NFKC", bl)
	212	+ if b != c:
	213	+ return c
	214	+ else:
	215	+ return al
	216	+
	217	+specials = {}
	218	+for k,v in table_b2.items():
	219	+ if map(ord, map_table_b2(unichr(k))) != v:
	220	+ specials[k] = v
	221	+
	222	+# B.3 should not add any additional special cases
	223	+assert specials == {}
	224	+
	225	+print """
	226	+def map_table_b2(a):
	227	+ al = map_table_b3(a)
	228	+ b = unicodedata.normalize("NFKC", al)
	229	+ bl = u"".join([map_table_b3(ch) for ch in b])
	230	+ c = unicodedata.normalize("NFKC", bl)
	231	+ if b != c:
	232	+ return c
	233	+ else:
	234	+ return al
	235	+"""
	236	+
	237	+# C.1.1 is a table with a single character
	238	+name, table = tables[0]
	239	+del tables[0]
	240	+assert name == "C.1.1"
	241	+assert table == {0x20:0x20}
	242	+
	243	+print """
	244	+def in_table_c11(code):
	245	+ return code == u" "
	246	+"""
	247	+
	248	+# C.1.2 is the rest of all space characters
	249	+name, table = tables[0]
	250	+del tables[0]
	251	+assert name == "C.1.2"
	252	+
	253	+# table = set(table.keys())
	254	+# Zs = set(gen_category(["Zs"])) - set([0x20])
	255	+# assert Zs == table
	256	+
	257	+print """
	258	+def in_table_c12(code):
	259	+ return unicodedata.category(code) == "Zs" and code != u" "
	260	+
	261	+def in_table_c11_c12(code):
	262	+ return unicodedata.category(code) == "Zs"
	263	+"""
	264	+
	265	+# C.2.1 ASCII control characters
	266	+name, table_c21 = tables[0]
	267	+del tables[0]
	268	+assert name == "C.2.1"
	269	+
	270	+Cc = set(gen_category(["Cc"]))
	271	+Cc_ascii = Cc & set(range(128))
	272	+table_c21 = set(table_c21.keys())
	273	+assert Cc_ascii == table_c21
	274	+
	275	+print """
	276	+def in_table_c21(code):
	277	+ return ord(code) < 128 and unicodedata.category(code) == "Cc"
	278	+"""
	279	+
	280	+# C.2.2 Non-ASCII control characters. It also includes
	281	+# a number of characters in category Cf.
	282	+name, table_c22 = tables[0]
	283	+del tables[0]
	284	+assert name == "C.2.2"
	285	+
	286	+Cc_nonascii = Cc - Cc_ascii
	287	+table_c22 = set(table_c22.keys())
	288	+assert len(Cc_nonascii - table_c22) == 0
	289	+
	290	+specials = list(table_c22 - Cc_nonascii)
	291	+specials.sort()
	292	+
	293	+print """c22_specials = """ + compact_set(specials) + """
	294	+def in_table_c22(code):
	295	+ c = ord(code)
	296	+ if c < 128: return False
	297	+ if unicodedata.category(code) == "Cc": return True
	298	+ return c in c22_specials
	299	+
	300	+def in_table_c21_c22(code):
	301	+ return unicodedata.category(code) == "Cc" or \\
	302	+ ord(code) in c22_specials
	303	+"""
	304	+
	305	+# C.3 Private use
	306	+name, table = tables[0]
	307	+del tables[0]
	308	+assert name == "C.3"
	309	+
	310	+Co = set(gen_category(["Co"]))
	311	+assert set(table.keys()) == Co
	312	+
	313	+print """
	314	+def in_table_c3(code):
	315	+ return unicodedata.category(code) == "Co"
	316	+"""
	317	+
	318	+# C.4 Non-character code points, xFFFE, xFFFF
	319	+# plus process internal codes
	320	+name, table = tables[0]
	321	+del tables[0]
	322	+assert name == "C.4"
	323	+
	324	+nonchar = set(range(0xFDD0,0xFDF0) +
	325	+ range(0xFFFE,0x110000,0x10000) +
	326	+ range(0xFFFF,0x110000,0x10000))
	327	+table = set(table.keys())
	328	+assert table == nonchar
	329	+
	330	+print """
	331	+def in_table_c4(code):
	332	+ c = ord(code)
	333	+ if c < 0xFDD0: return False
	334	+ if c < 0xFDF0: return True
	335	+ return (ord(code) & 0xFFFF) in (0xFFFE, 0xFFFF)
	336	+"""
	337	+
	338	+# C.5 Surrogate codes
	339	+name, table = tables[0]
	340	+del tables[0]
	341	+assert name == "C.5"
	342	+
	343	+Cs = set(gen_category(["Cs"]))
	344	+assert set(table.keys()) == Cs
	345	+
	346	+print """
	347	+def in_table_c5(code):
	348	+ return unicodedata.category(code) == "Cs"
	349	+"""
	350	+
	351	+# C.6 Inappropriate for plain text
	352	+name, table = tables[0]
	353	+del tables[0]
	354	+assert name == "C.6"
	355	+
	356	+table = table.keys()
	357	+table.sort()
	358	+
	359	+print """
	360	+c6_set = """ + compact_set(table) + """
	361	+def in_table_c6(code):
	362	+ return ord(code) in c6_set
	363	+"""
	364	+
	365	+# C.7 Inappropriate for canonical representation
	366	+name, table = tables[0]
	367	+del tables[0]
	368	+assert name == "C.7"
	369	+
	370	+table = table.keys()
	371	+table.sort()
	372	+
	373	+print """
	374	+c7_set = """ + compact_set(table) + """
	375	+def in_table_c7(code):
	376	+ return ord(code) in c7_set
	377	+"""
	378	+
	379	+# C.8 Change display properties or are deprecated
	380	+name, table = tables[0]
	381	+del tables[0]
	382	+assert name == "C.8"
	383	+
	384	+table = table.keys()
	385	+table.sort()
	386	+
	387	+print """
	388	+c8_set = """ + compact_set(table) + """
	389	+def in_table_c8(code):
	390	+ return ord(code) in c8_set
	391	+"""
	392	+
	393	+# C.9 Tagging characters
	394	+name, table = tables[0]
	395	+del tables[0]
	396	+assert name == "C.9"
	397	+
	398	+table = table.keys()
	399	+table.sort()
	400	+
	401	+print """
	402	+c9_set = """ + compact_set(table) + """
	403	+def in_table_c9(code):
	404	+ return ord(code) in c9_set
	405	+"""
	406	+
	407	+# D.1 Characters with bidirectional property "R" or "AL"
	408	+name, table = tables[0]
	409	+del tables[0]
	410	+assert name == "D.1"
	411	+
	412	+RandAL = set(gen_bidirectional(["R","AL"]))
	413	+assert set(table.keys()) == RandAL
	414	+
	415	+print """
	416	+def in_table_d1(code):
	417	+ return unicodedata.bidirectional(code) in ("R","AL")
	418	+"""
	419	+
	420	+# D.2 Characters with bidirectional property "L"
	421	+name, table = tables[0]
	422	+del tables[0]
	423	+assert name == "D.2"
	424	+
	425	+L = set(gen_bidirectional(["L"]))
	426	+assert set(table.keys()) == L
	427	+
	428	+print """
	429	+def in_table_d2(code):
	430	+ return unicodedata.bidirectional(code) == "L"
	431	+"""