History

My first initial commit with all of mine experimental changes.

Dmitry Kabak (userdima committed 13 years ago

Commit d37b88d1 parent 3f5b46618352 files · +2598093 −-8690Browse files

Showing the changes to java/pkg/sun/net/idn/StringPrep.java (1 of 18352 files). Show all

Unified · Split

java/pkg/sun/net/idn/StringPrep.java+487−0
@@ -0,0 +1,487 @@
1+/*
2+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
3+ *
4+ * This code is free software; you can redistribute it and/or modify it
5+ * under the terms of the GNU General Public License version 2 only, as
6+ * published by the Free Software Foundation. Oracle designates this
7+ * particular file as subject to the "Classpath" exception as provided
8+ * by Oracle in the LICENSE file that accompanied this code.
9+ *
10+ * This code is distributed in the hope that it will be useful, but WITHOUT
11+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13+ * version 2 for more details (a copy is included in the LICENSE file that
14+ * accompanied this code).
15+ *
16+ * You should have received a copy of the GNU General Public License version
17+ * 2 along with this work; if not, write to the Free Software Foundation,
18+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19+ *
20+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21+ * or visit www.oracle.com if you need additional information or have any
22+ * questions.
23+ */
24+/*
25+/*
26+ *******************************************************************************
27+ * Copyright (C) 2003-2004, International Business Machines Corporation and *
28+ * others. All Rights Reserved. *
29+ *******************************************************************************
30+ */
31+//
32+// CHANGELOG
33+// 2005-05-19 Edward Wang
34+// - copy this file from icu4jsrc_3_2/src/com/ibm/icu/text/StringPrep.java
35+// - move from package com.ibm.icu.text to package sun.net.idn
36+// - use ParseException instead of StringPrepParseException
37+// - change 'Normalizer.getUnicodeVersion()' to 'NormalizerImpl.getUnicodeVersion()'
38+// - remove all @deprecated tag to make compiler happy
39+// 2007-08-14 Martin Buchholz
40+// - remove redundant casts
41+//
42+package sun.net.idn;
43+
44+import java.io.BufferedInputStream;
45+import java.io.ByteArrayInputStream;
46+import java.io.IOException;
47+import java.io.InputStream;
48+import java.text.ParseException;
49+
50+import sun.text.Normalizer;
51+import sun.text.normalizer.CharTrie;
52+import sun.text.normalizer.Trie;
53+import sun.text.normalizer.NormalizerImpl;
54+import sun.text.normalizer.VersionInfo;
55+import sun.text.normalizer.UCharacter;
56+import sun.text.normalizer.UCharacterIterator;
57+import sun.text.normalizer.UTF16;
58+import sun.net.idn.UCharacterDirection;
59+import sun.net.idn.StringPrepDataReader;
60+
61+/**
62+ * StringPrep API implements the StingPrep framework as described by
63+ * <a href="http://www.ietf.org/rfc/rfc3454.txt">RFC 3454</a>.
64+ * StringPrep prepares Unicode strings for use in network protocols.
65+ * Profiles of StingPrep are set of rules and data according to which the
66+ * Unicode Strings are prepared. Each profiles contains tables which describe
67+ * how a code point should be treated. The tables are broadly classied into
68+ * <ul>
69+ * <li> Unassigned Table: Contains code points that are unassigned
70+ * in the Unicode Version supported by StringPrep. Currently
71+ * RFC 3454 supports Unicode 3.2. </li>
72+ * <li> Prohibited Table: Contains code points that are prohibted from
73+ * the output of the StringPrep processing function. </li>
74+ * <li> Mapping Table: Contains code ponts that are deleted from the output or case mapped. </li>
75+ * </ul>
76+ *
77+ * The procedure for preparing Unicode strings:
78+ * <ol>
79+ * <li> Map: For each character in the input, check if it has a mapping
80+ * and, if so, replace it with its mapping. </li>
81+ * <li> Normalize: Possibly normalize the result of step 1 using Unicode
82+ * normalization. </li>
83+ * <li> Prohibit: Check for any characters that are not allowed in the
84+ * output. If any are found, return an error.</li>
85+ * <li> Check bidi: Possibly check for right-to-left characters, and if
86+ * any are found, make sure that the whole string satisfies the
87+ * requirements for bidirectional strings. If the string does not
88+ * satisfy the requirements for bidirectional strings, return an
89+ * error. </li>
90+ * </ol>
91+ * @author Ram Viswanadha
92+ * @draft ICU 2.8
93+ */
94+public final class StringPrep {
95+ /**
96+ * Option to prohibit processing of unassigned code points in the input
97+ *
98+ * @see #prepare
99+ * @draft ICU 2.8
100+ */
101+ public static final int DEFAULT = 0x0000;
102+
103+ /**
104+ * Option to allow processing of unassigned code points in the input
105+ *
106+ * @see #prepare
107+ * @draft ICU 2.8
108+ */
109+ public static final int ALLOW_UNASSIGNED = 0x0001;
110+
111+ private static final int UNASSIGNED = 0x0000;
112+ private static final int MAP = 0x0001;
113+ private static final int PROHIBITED = 0x0002;
114+ private static final int DELETE = 0x0003;
115+ private static final int TYPE_LIMIT = 0x0004;
116+
117+ private static final int NORMALIZATION_ON = 0x0001;
118+ private static final int CHECK_BIDI_ON = 0x0002;
119+
120+ private static final int TYPE_THRESHOLD = 0xFFF0;
121+ private static final int MAX_INDEX_VALUE = 0x3FBF; /*16139*/
122+ private static final int MAX_INDEX_TOP_LENGTH = 0x0003;
123+
124+ /* indexes[] value names */
125+ private static final int INDEX_TRIE_SIZE = 0; /* number of bytes in normalization trie */
126+ private static final int INDEX_MAPPING_DATA_SIZE = 1; /* The array that contains the mapping */
127+ private static final int NORM_CORRECTNS_LAST_UNI_VERSION = 2; /* The index of Unicode version of last entry in NormalizationCorrections.txt */
128+ private static final int ONE_UCHAR_MAPPING_INDEX_START = 3; /* The starting index of 1 UChar mapping index in the mapping data array */
129+ private static final int TWO_UCHARS_MAPPING_INDEX_START = 4; /* The starting index of 2 UChars mapping index in the mapping data array */
130+ private static final int THREE_UCHARS_MAPPING_INDEX_START = 5;
131+ private static final int FOUR_UCHARS_MAPPING_INDEX_START = 6;
132+ private static final int OPTIONS = 7; /* Bit set of options to turn on in the profile */
133+ private static final int INDEX_TOP = 16; /* changing this requires a new formatVersion */
134+
135+
136+ /**
137+ * Default buffer size of datafile
138+ */
139+ private static final int DATA_BUFFER_SIZE = 25000;
140+
141+ /* Wrappers for Trie implementations */
142+ private static final class StringPrepTrieImpl implements Trie.DataManipulate{
143+ private CharTrie sprepTrie = null;
144+ /**
145+ * Called by com.ibm.icu.util.Trie to extract from a lead surrogate's
146+ * data the index array offset of the indexes for that lead surrogate.
147+ * @param property data value for a surrogate from the trie, including
148+ * the folding offset
149+ * @return data offset or 0 if there is no data for the lead surrogate
150+ */
151+ public int getFoldingOffset(int value){
152+ return value;
153+ }
154+ }
155+
156+ // CharTrie implmentation for reading the trie data
157+ private StringPrepTrieImpl sprepTrieImpl;
158+ // Indexes read from the data file
159+ private int[] indexes;
160+ // mapping data read from the data file
161+ private char[] mappingData;
162+ // format version of the data file
163+ private byte[] formatVersion;
164+ // the version of Unicode supported by the data file
165+ private VersionInfo sprepUniVer;
166+ // the Unicode version of last entry in the
167+ // NormalizationCorrections.txt file if normalization
168+ // is turned on
169+ private VersionInfo normCorrVer;
170+ // Option to turn on Normalization
171+ private boolean doNFKC;
172+ // Option to turn on checking for BiDi rules
173+ private boolean checkBiDi;
174+
175+
176+ private char getCodePointValue(int ch){
177+ return sprepTrieImpl.sprepTrie.getCodePointValue(ch);
178+ }
179+
180+ private static VersionInfo getVersionInfo(int comp){
181+ int micro = comp & 0xFF;
182+ int milli =(comp >> 8) & 0xFF;
183+ int minor =(comp >> 16) & 0xFF;
184+ int major =(comp >> 24) & 0xFF;
185+ return VersionInfo.getInstance(major,minor,milli,micro);
186+ }
187+ private static VersionInfo getVersionInfo(byte[] version){
188+ if(version.length != 4){
189+ return null;
190+ }
191+ return VersionInfo.getInstance((int)version[0],(int) version[1],(int) version[2],(int) version[3]);
192+ }
193+ /**
194+ * Creates an StringPrep object after reading the input stream.
195+ * The object does not hold a reference to the input steam, so the stream can be
196+ * closed after the method returns.
197+ *
198+ * @param inputStream The stream for reading the StringPrep profile binarySun
199+ * @throws IOException
200+ * @draft ICU 2.8
201+ */
202+ public StringPrep(InputStream inputStream) throws IOException{
203+
204+ BufferedInputStream b = new BufferedInputStream(inputStream,DATA_BUFFER_SIZE);
205+
206+ StringPrepDataReader reader = new StringPrepDataReader(b);
207+
208+ // read the indexes
209+ indexes = reader.readIndexes(INDEX_TOP);
210+
211+ byte[] sprepBytes = new byte[indexes[INDEX_TRIE_SIZE]];
212+
213+
214+ //indexes[INDEX_MAPPING_DATA_SIZE] store the size of mappingData in bytes
215+ mappingData = new char[indexes[INDEX_MAPPING_DATA_SIZE]/2];
216+ // load the rest of the data data and initialize the data members
217+ reader.read(sprepBytes,mappingData);
218+
219+ sprepTrieImpl = new StringPrepTrieImpl();
220+ sprepTrieImpl.sprepTrie = new CharTrie( new ByteArrayInputStream(sprepBytes),sprepTrieImpl );
221+
222+ // get the data format version
223+ formatVersion = reader.getDataFormatVersion();
224+
225+ // get the options
226+ doNFKC = ((indexes[OPTIONS] & NORMALIZATION_ON) > 0);
227+ checkBiDi = ((indexes[OPTIONS] & CHECK_BIDI_ON) > 0);
228+ sprepUniVer = getVersionInfo(reader.getUnicodeVersion());
229+ normCorrVer = getVersionInfo(indexes[NORM_CORRECTNS_LAST_UNI_VERSION]);
230+ VersionInfo normUniVer = NormalizerImpl.getUnicodeVersion();
231+ if(normUniVer.compareTo(sprepUniVer) < 0 && /* the Unicode version of SPREP file must be less than the Unicode Vesion of the normalization data */
232+ normUniVer.compareTo(normCorrVer) < 0 && /* the Unicode version of the NormalizationCorrections.txt file should be less than the Unicode Vesion of the normalization data */
233+ ((indexes[OPTIONS] & NORMALIZATION_ON) > 0) /* normalization turned on*/
234+ ){
235+ throw new IOException("Normalization Correction version not supported");
236+ }
237+ b.close();
238+ }
239+
240+ private static final class Values{
241+ boolean isIndex;
242+ int value;
243+ int type;
244+ public void reset(){
245+ isIndex = false;
246+ value = 0;
247+ type = -1;
248+ }
249+ }
250+
251+ private static final void getValues(char trieWord,Values values){
252+ values.reset();
253+ if(trieWord == 0){
254+ /*
255+ * Initial value stored in the mapping table
256+ * just return TYPE_LIMIT .. so that
257+ * the source codepoint is copied to the destination
258+ */
259+ values.type = TYPE_LIMIT;
260+ }else if(trieWord >= TYPE_THRESHOLD){
261+ values.type = (trieWord - TYPE_THRESHOLD);
262+ }else{
263+ /* get the type */
264+ values.type = MAP;
265+ /* ascertain if the value is index or delta */
266+ if((trieWord & 0x02)>0){
267+ values.isIndex = true;
268+ values.value = trieWord >> 2; //mask off the lower 2 bits and shift
269+
270+ }else{
271+ values.isIndex = false;
272+ values.value = (trieWord<<16)>>16;
273+ values.value = (values.value >> 2);
274+
275+ }
276+
277+ if((trieWord>>2) == MAX_INDEX_VALUE){
278+ values.type = DELETE;
279+ values.isIndex = false;
280+ values.value = 0;
281+ }
282+ }
283+ }
284+
285+
286+
287+ private StringBuffer map( UCharacterIterator iter, int options)
288+ throws ParseException {
289+
290+ Values val = new Values();
291+ char result = 0;
292+ int ch = UCharacterIterator.DONE;
293+ StringBuffer dest = new StringBuffer();
294+ boolean allowUnassigned = ((options & ALLOW_UNASSIGNED)>0);
295+
296+ while((ch=iter.nextCodePoint())!= UCharacterIterator.DONE){
297+
298+ result = getCodePointValue(ch);
299+ getValues(result,val);
300+
301+ // check if the source codepoint is unassigned
302+ if(val.type == UNASSIGNED && allowUnassigned == false){
303+ throw new ParseException("An unassigned code point was found in the input " +
304+ iter.getText(), iter.getIndex());
305+ }else if((val.type == MAP)){
306+ int index, length;
307+
308+ if(val.isIndex){
309+ index = val.value;
310+ if(index >= indexes[ONE_UCHAR_MAPPING_INDEX_START] &&
311+ index < indexes[TWO_UCHARS_MAPPING_INDEX_START]){
312+ length = 1;
313+ }else if(index >= indexes[TWO_UCHARS_MAPPING_INDEX_START] &&
314+ index < indexes[THREE_UCHARS_MAPPING_INDEX_START]){
315+ length = 2;
316+ }else if(index >= indexes[THREE_UCHARS_MAPPING_INDEX_START] &&
317+ index < indexes[FOUR_UCHARS_MAPPING_INDEX_START]){
318+ length = 3;
319+ }else{
320+ length = mappingData[index++];
321+ }
322+ /* copy mapping to destination */
323+ dest.append(mappingData,index,length);
324+ continue;
325+
326+ }else{
327+ ch -= val.value;
328+ }
329+ }else if(val.type == DELETE){
330+ // just consume the codepoint and contine
331+ continue;
332+ }
333+ //copy the source into destination
334+ UTF16.append(dest,ch);
335+ }
336+
337+ return dest;
338+ }
339+
340+
341+ private StringBuffer normalize(StringBuffer src){
342+ /*
343+ * Option UNORM_BEFORE_PRI_29:
344+ *
345+ * IDNA as interpreted by IETF members (see unicode mailing list 2004H1)
346+ * requires strict adherence to Unicode 3.2 normalization,
347+ * including buggy composition from before fixing Public Review Issue #29.
348+ * Note that this results in some valid but nonsensical text to be
349+ * either corrupted or rejected, depending on the text.
350+ * See http://www.unicode.org/review/resolved-pri.html#pri29
351+ * See unorm.cpp and cnormtst.c
352+ */
353+ return new StringBuffer(
354+ Normalizer.normalize(
355+ src.toString(),
356+ java.text.Normalizer.Form.NFKC,
357+ Normalizer.UNICODE_3_2|NormalizerImpl.BEFORE_PRI_29));
358+ }
359+ /*
360+ boolean isLabelSeparator(int ch){
361+ int result = getCodePointValue(ch);
362+ if( (result & 0x07) == LABEL_SEPARATOR){
363+ return true;
364+ }
365+ return false;
366+ }
367+ */
368+ /*
369+ 1) Map -- For each character in the input, check if it has a mapping
370+ and, if so, replace it with its mapping.
371+
372+ 2) Normalize -- Possibly normalize the result of step 1 using Unicode
373+ normalization.
374+
375+ 3) Prohibit -- Check for any characters that are not allowed in the
376+ output. If any are found, return an error.
377+
378+ 4) Check bidi -- Possibly check for right-to-left characters, and if
379+ any are found, make sure that the whole string satisfies the
380+ requirements for bidirectional strings. If the string does not
381+ satisfy the requirements for bidirectional strings, return an
382+ error.
383+ [Unicode3.2] defines several bidirectional categories; each character
384+ has one bidirectional category assigned to it. For the purposes of
385+ the requirements below, an "RandALCat character" is a character that
386+ has Unicode bidirectional categories "R" or "AL"; an "LCat character"
387+ is a character that has Unicode bidirectional category "L". Note
388+
389+
390+ that there are many characters which fall in neither of the above
391+ definitions; Latin digits (<U+0030> through <U+0039>) are examples of
392+ this because they have bidirectional category "EN".
393+
394+ In any profile that specifies bidirectional character handling, all
395+ three of the following requirements MUST be met:
396+
397+ 1) The characters in section 5.8 MUST be prohibited.
398+
399+ 2) If a string contains any RandALCat character, the string MUST NOT
400+ contain any LCat character.
401+
402+ 3) If a string contains any RandALCat character, a RandALCat
403+ character MUST be the first character of the string, and a
404+ RandALCat character MUST be the last character of the string.
405+ */
406+ /**
407+ * Prepare the input buffer for use in applications with the given profile. This operation maps, normalizes(NFKC),
408+ * checks for prohited and BiDi characters in the order defined by RFC 3454
409+ * depending on the options specified in the profile.
410+ *
411+ * @param src A UCharacterIterator object containing the source string
412+ * @param options A bit set of options:
413+ *
414+ * - StringPrep.NONE Prohibit processing of unassigned code points in the input
415+ *
416+ * - StringPrep.ALLOW_UNASSIGNED Treat the unassigned code points are in the input
417+ * as normal Unicode code points.
418+ *
419+ * @return StringBuffer A StringBuffer containing the output
420+ * @throws ParseException
421+ * @draft ICU 2.8
422+ */
423+ public StringBuffer prepare(UCharacterIterator src, int options)
424+ throws ParseException{
425+
426+ // map
427+ StringBuffer mapOut = map(src,options);
428+ StringBuffer normOut = mapOut;// initialize
429+
430+ if(doNFKC){
431+ // normalize
432+ normOut = normalize(mapOut);
433+ }
434+
435+ int ch;
436+ char result;
437+ UCharacterIterator iter = UCharacterIterator.getInstance(normOut);
438+ Values val = new Values();
439+ int direction=UCharacterDirection.CHAR_DIRECTION_COUNT,
440+ firstCharDir=UCharacterDirection.CHAR_DIRECTION_COUNT;
441+ int rtlPos=-1, ltrPos=-1;
442+ boolean rightToLeft=false, leftToRight=false;
443+
444+ while((ch=iter.nextCodePoint())!= UCharacterIterator.DONE){
445+ result = getCodePointValue(ch);
446+ getValues(result,val);
447+
448+ if(val.type == PROHIBITED ){
449+ throw new ParseException("A prohibited code point was found in the input" +
450+ iter.getText(), val.value);
451+ }
452+
453+ direction = UCharacter.getDirection(ch);
454+ if(firstCharDir == UCharacterDirection.CHAR_DIRECTION_COUNT){
455+ firstCharDir = direction;
456+ }
457+ if(direction == UCharacterDirection.LEFT_TO_RIGHT){
458+ leftToRight = true;
459+ ltrPos = iter.getIndex()-1;
460+ }
461+ if(direction == UCharacterDirection.RIGHT_TO_LEFT || direction == UCharacterDirection.RIGHT_TO_LEFT_ARABIC){
462+ rightToLeft = true;
463+ rtlPos = iter.getIndex()-1;
464+ }
465+ }
466+ if(checkBiDi == true){
467+ // satisfy 2
468+ if( leftToRight == true && rightToLeft == true){
469+ throw new ParseException("The input does not conform to the rules for BiDi code points." +
470+ iter.getText(),
471+ (rtlPos>ltrPos) ? rtlPos : ltrPos);
472+ }
473+
474+ //satisfy 3
475+ if( rightToLeft == true &&
476+ !((firstCharDir == UCharacterDirection.RIGHT_TO_LEFT || firstCharDir == UCharacterDirection.RIGHT_TO_LEFT_ARABIC) &&
477+ (direction == UCharacterDirection.RIGHT_TO_LEFT || direction == UCharacterDirection.RIGHT_TO_LEFT_ARABIC))
478+ ){
479+ throw new ParseException("The input does not conform to the rules for BiDi code points." +
480+ iter.getText(),
481+ (rtlPos>ltrPos) ? rtlPos : ltrPos);
482+ }
483+ }
484+ return normOut;
485+
486+ }
487+}