/* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as * published by the Free Software Foundation.  Oracle designates this * particular file as subject to the "Classpath" exception as provided * by Oracle in the LICENSE file that accompanied this code. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA * or visit www.oracle.com if you need additional information or have any * questions. *//* ******************************************************************************* * (C) Copyright IBM Corp. and others, 1996-2009 - All Rights Reserved         * *                                                                             * * The original version of this source code and documentation is copyrighted   * * and owned by IBM, These materials are provided under terms of a License     * * Agreement between IBM and Sun. This technology is protected by multiple     * * US and International patents. This notice and attribution to IBM may not    * * to removed.                                                                 * ******************************************************************************* */ package sun.text.normalizer; import java.io.BufferedInputStream;import java.io.ByteArrayInputStream;import java.io.IOException;import java.io.BufferedInputStream;import java.io.InputStream; /** * @author  Ram Viswanadha */public final class NormalizerImpl {    // Static block for the class to initialize its own self    static final NormalizerImpl IMPL;     static    {        try        {            IMPL = new NormalizerImpl();        }        catch (Exception e)        {            throw new RuntimeException(e.getMessage());        }    }     static final int UNSIGNED_BYTE_MASK =0xFF;    static final long UNSIGNED_INT_MASK = 0xffffffffL;    /*     * This new implementation of the normalization code loads its data from     * unorm.icu, which is generated with the gennorm tool.     * The format of that file is described at the end of this file.     */    private static final String DATA_FILE_NAME = "/sun/text/resources/unorm.icu";     // norm32 value constants     // quick check flags 0..3 set mean "no" for their forms    public static final int QC_NFC=0x11;          /* no|maybe */    public static final int QC_NFKC=0x22;         /* no|maybe */    public static final int QC_NFD=4;             /* no */    public static final int QC_NFKD=8;            /* no */     public static final int QC_ANY_NO=0xf;     /* quick check flags 4..5 mean "maybe" for their forms;     * test flags>=QC_MAYBE     */    public static final int QC_MAYBE=0x10;    public static final int QC_ANY_MAYBE=0x30;     public static final int QC_MASK=0x3f;     private static final int COMBINES_FWD=0x40;    private static final int COMBINES_BACK=0x80;    public  static final int COMBINES_ANY=0xc0;    // UnicodeData.txt combining class in bits 15.    private static final int CC_SHIFT=8;    public  static final int CC_MASK=0xff00;    // 16 bits for the index to UChars and other extra data    private static final int EXTRA_SHIFT=16;     /* norm32 value constants using >16 bits */    private static final long  MIN_SPECIAL    =  (long)(0xfc000000 & UNSIGNED_INT_MASK);    private static final long  SURROGATES_TOP =  (long)(0xfff00000 & UNSIGNED_INT_MASK);    private static final long  MIN_HANGUL     =  (long)(0xfff00000 & UNSIGNED_INT_MASK);//  private static final long  MIN_JAMO_V     =  (long)(0xfff20000 & UNSIGNED_INT_MASK);    private static final long  JAMO_V_TOP     =  (long)(0xfff30000 & UNSIGNED_INT_MASK);      /* indexes[] value names */    /* number of bytes in normalization trie */    static final int INDEX_TRIE_SIZE           = 0;    /* number of chars in extra data */    static final int INDEX_CHAR_COUNT           = 1;    /* number of uint16_t words for combining data */    static final int INDEX_COMBINE_DATA_COUNT = 2;    /* first code point with quick check NFC NO/MAYBE */    public static final int INDEX_MIN_NFC_NO_MAYBE   = 6;    /* first code point with quick check NFKC NO/MAYBE */    public static final int INDEX_MIN_NFKC_NO_MAYBE  = 7;    /* first code point with quick check NFD NO/MAYBE */    public static final int INDEX_MIN_NFD_NO_MAYBE   = 8;    /* first code point with quick check NFKD NO/MAYBE */    public static final int INDEX_MIN_NFKD_NO_MAYBE  = 9;    /* number of bytes in FCD trie */    static final int INDEX_FCD_TRIE_SIZE      = 10;    /* number of bytes in the auxiliary trie */    static final int INDEX_AUX_TRIE_SIZE      = 11;    /* changing this requires a new formatVersion */    static final int INDEX_TOP                = 32;      /* AUX constants */    /* value constants for auxTrie */    private static final int AUX_UNSAFE_SHIFT           = 11;    private static final int AUX_COMP_EX_SHIFT           = 10;    private static final int AUX_NFC_SKIPPABLE_F_SHIFT = 12;     private static final int AUX_MAX_FNC          =   ((int)1<<AUX_COMP_EX_SHIFT);    private static final int AUX_UNSAFE_MASK      =   (int)((1<<AUX_UNSAFE_SHIFT) & UNSIGNED_INT_MASK);    private static final int AUX_FNC_MASK         =   (int)((AUX_MAX_FNC-1) & UNSIGNED_INT_MASK);    private static final int AUX_COMP_EX_MASK     =   (int)((1<<AUX_COMP_EX_SHIFT) & UNSIGNED_INT_MASK);    private static final long AUX_NFC_SKIP_F_MASK =   ((UNSIGNED_INT_MASK&1)<<AUX_NFC_SKIPPABLE_F_SHIFT);     private static final int MAX_BUFFER_SIZE                    = 20;     /*******************************/     /* Wrappers for Trie implementations */    static final class NormTrieImpl implements Trie.DataManipulate{        static IntTrie normTrie= null;       /**        * Called by com.ibm.icu.util.Trie to extract from a lead surrogate's        * data the index array offset of the indexes for that lead surrogate.        * @param property data value for a surrogate from the trie, including        *         the folding offset        * @return data offset or 0 if there is no data for the lead surrogate        */        /* normTrie: 32-bit trie result may contain a special extraData index with the folding offset */        public int getFoldingOffset(int value){            return  BMP_INDEX_LENGTH+                    ((value>>(EXTRA_SHIFT-SURROGATE_BLOCK_BITS))&                    (0x3ff<<SURROGATE_BLOCK_BITS));        }     }    static final class FCDTrieImpl implements Trie.DataManipulate{        static CharTrie fcdTrie=null;       /**        * Called by com.ibm.icu.util.Trie to extract from a lead surrogate's        * data the index array offset of the indexes for that lead surrogate.        * @param property data value for a surrogate from the trie, including        *         the folding offset        * @return data offset or 0 if there is no data for the lead surrogate        */        /* fcdTrie: the folding offset is the lead FCD value itself */        public int getFoldingOffset(int value){            return value;        }    }     static final class AuxTrieImpl implements Trie.DataManipulate{        static CharTrie auxTrie = null;       /**        * Called by com.ibm.icu.util.Trie to extract from a lead surrogate's        * data the index array offset of the indexes for that lead surrogate.        * @param property data value for a surrogate from the trie, including        *        the folding offset        * @return data offset or 0 if there is no data for the lead surrogate        */        /* auxTrie: the folding offset is in bits 9..0 of the 16-bit trie result */        public int getFoldingOffset(int value){            return (int)(value &AUX_FNC_MASK)<<SURROGATE_BLOCK_BITS;        }    }     /****************************************************/      private static FCDTrieImpl fcdTrieImpl;    private static NormTrieImpl normTrieImpl;    private static AuxTrieImpl auxTrieImpl;    private static int[] indexes;    private static char[] combiningTable;    private static char[] extraData;     private static boolean isDataLoaded;    private static boolean isFormatVersion_2_1;    private static boolean isFormatVersion_2_2;    private static byte[] unicodeVersion;     /**     * Default buffer size of datafile     */    private static final int DATA_BUFFER_SIZE = 25000;     /**     * FCD check: everything below this code point is known to have a 0     * lead combining class     */    public static final int MIN_WITH_LEAD_CC=0x300;      /**     * Bit 7 of the length byte for a decomposition string in extra data is     * a flag indicating whether the decomposition string is     * preceded by a 16-bit word with the leading and trailing cc     * of the decomposition (like for A-umlaut);     * if not, then both cc's are zero (like for compatibility ideographs).     */    private static final int DECOMP_FLAG_LENGTH_HAS_CC=0x80;    /**     * Bits 6..0 of the length byte contain the actual length.     */    private static final int DECOMP_LENGTH_MASK=0x7f;     /** Length of the BMP portion of the index (stage 1) array. */    private static final int BMP_INDEX_LENGTH=0x10000>>Trie.INDEX_STAGE_1_SHIFT_;    /** Number of bits of a trail surrogate that are used in index table     * lookups.     */    private static final int SURROGATE_BLOCK_BITS=10-Trie.INDEX_STAGE_1_SHIFT_;     // public utility   public static int getFromIndexesArr(int index){        return indexes[index];   }    // protected constructor ---------------------------------------------     /**    * Constructor    * @exception thrown when data reading fails or data corrupted    */    private NormalizerImpl() throws IOException {        //data should be loaded only once        if(!isDataLoaded){             // jar access            InputStream i = ICUData.getRequiredStream(DATA_FILE_NAME);            BufferedInputStream b = new BufferedInputStream(i,DATA_BUFFER_SIZE);            NormalizerDataReader reader = new NormalizerDataReader(b);             // read the indexes            indexes = reader.readIndexes(NormalizerImpl.INDEX_TOP);             byte[] normBytes = new byte[indexes[NormalizerImpl.INDEX_TRIE_SIZE]];             int combiningTableTop = indexes[NormalizerImpl.INDEX_COMBINE_DATA_COUNT];            combiningTable = new char[combiningTableTop];             int extraDataTop = indexes[NormalizerImpl.INDEX_CHAR_COUNT];            extraData = new char[extraDataTop];             byte[] fcdBytes = new byte[indexes[NormalizerImpl.INDEX_FCD_TRIE_SIZE]];            byte[] auxBytes = new byte[indexes[NormalizerImpl.INDEX_AUX_TRIE_SIZE]];             fcdTrieImpl = new FCDTrieImpl();            normTrieImpl = new NormTrieImpl();            auxTrieImpl = new AuxTrieImpl();             // load the rest of the data data and initialize the data members            reader.read(normBytes, fcdBytes,auxBytes, extraData, combiningTable);             NormTrieImpl.normTrie = new IntTrie( new ByteArrayInputStream(normBytes),normTrieImpl );            FCDTrieImpl.fcdTrie   = new CharTrie( new ByteArrayInputStream(fcdBytes),fcdTrieImpl  );            AuxTrieImpl.auxTrie   = new CharTrie( new ByteArrayInputStream(auxBytes),auxTrieImpl  );             // we reached here without any exceptions so the data is fully            // loaded set the variable to true            isDataLoaded = true;             // get the data format version            byte[] formatVersion = reader.getDataFormatVersion();             isFormatVersion_2_1 =( formatVersion[0]>2                                    ||                                   (formatVersion[0]==2 && formatVersion[1]>=1)                                 );            isFormatVersion_2_2 =( formatVersion[0]>2                                    ||                                   (formatVersion[0]==2 && formatVersion[1]>=2)                                 );            unicodeVersion = reader.getUnicodeVersion();            b.close();        }    }     /* ---------------------------------------------------------------------- */     /* Korean Hangul and Jamo constants */     public static final int JAMO_L_BASE=0x1100;     /* "lead" jamo */    public static final int JAMO_V_BASE=0x1161;     /* "vowel" jamo */    public static final int JAMO_T_BASE=0x11a7;     /* "trail" jamo */     public static final int HANGUL_BASE=0xac00;     public static final int JAMO_L_COUNT=19;    public static final int JAMO_V_COUNT=21;    public static final int JAMO_T_COUNT=28;    public  static final int HANGUL_COUNT=JAMO_L_COUNT*JAMO_V_COUNT*JAMO_T_COUNT;     private static boolean isHangulWithoutJamoT(char c) {        c-=HANGUL_BASE;        return c<HANGUL_COUNT && c%JAMO_T_COUNT==0;    }     /* norm32 helpers */     /* is this a norm32 with a regular index? */    private static boolean isNorm32Regular(long norm32) {        return norm32<MIN_SPECIAL;    }     /* is this a norm32 with a special index for a lead surrogate? */    private static boolean isNorm32LeadSurrogate(long norm32) {        return MIN_SPECIAL<=norm32 && norm32<SURROGATES_TOP;    }     /* is this a norm32 with a special index for a Hangul syllable or a Jamo? */    private static boolean isNorm32HangulOrJamo(long norm32) {        return norm32>=MIN_HANGUL;    }     /*     * Given norm32 for Jamo V or T,     * is this a Jamo V?     */    private static boolean isJamoVTNorm32JamoV(long norm32) {        return norm32<JAMO_V_TOP;    }     /* data access primitives ----------------------------------------------- */     public static long/*unsigned*/ getNorm32(char c) {        return ((UNSIGNED_INT_MASK) & (NormTrieImpl.normTrie.getLeadValue(c)));    }     public static long/*unsigned*/ getNorm32FromSurrogatePair(long norm32,                                                               char c2) {        /*         * the surrogate index in norm32 stores only the number of the surrogate         * index block see gennorm/store.c/getFoldedNormValue()         */        return ((UNSIGNED_INT_MASK) &                    NormTrieImpl.normTrie.getTrailValue((int)norm32, c2));    }    ///CLOVER:OFF    private static long getNorm32(int c){        return (UNSIGNED_INT_MASK&(NormTrieImpl.normTrie.getCodePointValue(c)));    }     /*     * get a norm32 from text with complete code points     * (like from decompositions)     */    private static long/*unsigned*/ getNorm32(char[] p,int start,                                              int/*unsigned*/ mask) {        long/*unsigned*/ norm32= getNorm32(p[start]);        if(((norm32&mask)>0) && isNorm32LeadSurrogate(norm32)) {            /* *p is a lead surrogate, get the real norm32 */            norm32=getNorm32FromSurrogatePair(norm32, p[start+1]);        }        return norm32;    }     //// for StringPrep    public static VersionInfo getUnicodeVersion(){        return VersionInfo.getInstance(unicodeVersion[0], unicodeVersion[1],                                       unicodeVersion[2], unicodeVersion[3]);    }     public static char    getFCD16(char c) {        return  FCDTrieImpl.fcdTrie.getLeadValue(c);    }     public static char getFCD16FromSurrogatePair(char fcd16, char c2) {        /* the surrogate index in fcd16 is an absolute offset over the         * start of stage 1         * */        return FCDTrieImpl.fcdTrie.getTrailValue(fcd16, c2);    }    public static int getFCD16(int c) {        return  FCDTrieImpl.fcdTrie.getCodePointValue(c);    }     private static int getExtraDataIndex(long norm32) {        return (int)(norm32>>EXTRA_SHIFT);    }     private static final class DecomposeArgs{        int /*unsigned byte*/ cc;        int /*unsigned byte*/ trailCC;        int length;    }    /**     *     * get the canonical or compatibility decomposition for one character     *     * @return index into the extraData array     */    private static int/*index*/ decompose(long/*unsigned*/ norm32,                                          int/*unsigned*/ qcMask,                                          DecomposeArgs args) {        int p= getExtraDataIndex(norm32);        args.length=extraData[p++];         if((norm32&qcMask&QC_NFKD)!=0 && args.length>=0x100) {            /* use compatibility decomposition, skip canonical data */            p+=((args.length>>7)&1)+(args.length&DECOMP_LENGTH_MASK);            args.length>>=8;        }         if((args.length&DECOMP_FLAG_LENGTH_HAS_CC)>0) {            /* get the lead and trail cc's */            char bothCCs=extraData[p++];            args.cc=(UNSIGNED_BYTE_MASK) & (bothCCs>>8);            args.trailCC=(UNSIGNED_BYTE_MASK) & bothCCs;        } else {            /* lead and trail cc's are both 0 */            args.cc=args.trailCC=0;        }         args.length&=DECOMP_LENGTH_MASK;        return p;    }      /**     * get the canonical decomposition for one character     * @return index into the extraData array     */    private static int decompose(long/*unsigned*/ norm32,                                 DecomposeArgs args) {         int p= getExtraDataIndex(norm32);        args.length=extraData[p++];         if((args.length&DECOMP_FLAG_LENGTH_HAS_CC)>0) {            /* get the lead and trail cc's */            char bothCCs=extraData[p++];            args.cc=(UNSIGNED_BYTE_MASK) & (bothCCs>>8);            args.trailCC=(UNSIGNED_BYTE_MASK) & bothCCs;        } else {            /* lead and trail cc's are both 0 */            args.cc=args.trailCC=0;        }         args.length&=DECOMP_LENGTH_MASK;        return p;    }      private static final class NextCCArgs{        char[] source;        int next;        int limit;        char c;        char c2;    }     /*     * get the combining class of (c, c2)= args.source[args.next++]     * before: args.next<args.limit  after: args.next<=args.limit     * if only one code unit is used, then c2==0     */    private static int /*unsigned byte*/ getNextCC(NextCCArgs args) {        long /*unsigned*/ norm32;         args.c=args.source[args.next++];         norm32= getNorm32(args.c);        if((norm32 & CC_MASK)==0) {            args.c2=0;            return 0;        } else {            if(!isNorm32LeadSurrogate(norm32)) {                args.c2=0;            } else {                /* c is a lead surrogate, get the real norm32 */

Lines 1–500 of 2736. Load more