// Written in the D programming language. /**Classes and functions for handling and transcoding between various encodings. For cases where the encoding is known at compile-time, functions are providedfor arbitrary encoding and decoding of characters, arbitrary transcodingbetween strings of different type, as well as validation and sanitization. Encodings currently supported are UTF-8, UTF-16, UTF-32, ASCII, ISO-8859-1(also known as LATIN-1), ISO-8859-2 (LATIN-2), WINDOWS-1250, WINDOWS-1251and WINDOWS-1252. $(SCRIPT inhibitQuickIndex = 1;)$(DIVC quickindex,$(BOOKTABLE,$(TR $(TH Category) $(TH Functions))$(TR $(TD Decode) $(TD    $(LREF codePoints)    $(LREF decode)    $(LREF decodeReverse)    $(LREF safeDecode)))$(TR $(TD Conversion) $(TD    $(LREF codeUnits)    $(LREF sanitize)    $(LREF transcode)))$(TR $(TD Classification) $(TD    $(LREF canEncode)    $(LREF isValid)    $(LREF isValidCodePoint)    $(LREF isValidCodeUnit)))$(TR $(TD BOM) $(TD    $(LREF BOM)    $(LREF BOMSeq)    $(LREF getBOM)    $(LREF utfBOM)))$(TR $(TD Length &amp; Index) $(TD    $(LREF firstSequence)    $(LREF encodedLength)    $(LREF index)    $(LREF lastSequence)    $(LREF validLength)))$(TR $(TD Encoding schemes) $(TD    $(LREF encodingName)    $(LREF EncodingScheme)    $(LREF EncodingSchemeASCII)    $(LREF EncodingSchemeLatin1)    $(LREF EncodingSchemeLatin2)    $(LREF EncodingSchemeUtf16Native)    $(LREF EncodingSchemeUtf32Native)    $(LREF EncodingSchemeUtf8)    $(LREF EncodingSchemeWindows1250)    $(LREF EncodingSchemeWindows1251)    $(LREF EncodingSchemeWindows1252)))$(TR $(TD Representation) $(TD    $(LREF AsciiChar)    $(LREF AsciiString)    $(LREF Latin1Char)    $(LREF Latin1String)    $(LREF Latin2Char)    $(LREF Latin2String)    $(LREF Windows1250Char)    $(LREF Windows1250String)    $(LREF Windows1251Char)    $(LREF Windows1251String)    $(LREF Windows1252Char)    $(LREF Windows1252String)))$(TR $(TD Exceptions) $(TD    $(LREF INVALID_SEQUENCE)    $(LREF EncodingException))))) For cases where the encoding is not known at compile-time, but isknown at run-time, the abstract class $(LREF EncodingScheme)and its subclasses is provided.  To construct a run-time encoder/decoder,one does e.g. ----------------------------------------------------auto e = EncodingScheme.create("utf-8");---------------------------------------------------- This library supplies $(LREF EncodingScheme) subclasses for ASCII,ISO-8859-1 (also known as LATIN-1), ISO-8859-2 (LATIN-2), WINDOWS-1250,WINDOWS-1251, WINDOWS-1252, UTF-8, and (on little-endian architectures)UTF-16LE and UTF-32LE; or (on big-endian architectures) UTF-16BE and UTF-32BE. This library provides a mechanism whereby other modules may add $(LREFEncodingScheme) subclasses for any other encoding. Copyright: Copyright Janice Caron 2008 - 2009.License:   $(HTTP www.boost.org/LICENSE_1_0.txt, Boost License 1.0).Authors:   Janice CaronSource:    $(PHOBOSSRC std/encoding.d)*//*         Copyright Janice Caron 2008 - 2009.Distributed under the Boost Software License, Version 1.0.   (See accompanying file LICENSE_1_0.txt or copy at         http://www.boost.org/LICENSE_1_0.txt)*/module std.encoding; import std.range.primitives;import std.traits;import std.typecons; @system unittest{    static ubyte[][] validStrings =    [        // Plain ASCII        cast(ubyte[])"hello",         // First possible sequence of a certain length        [ 0x00 ],                       // U+00000000   one byte        [ 0xC2, 0x80 ],                 // U+00000080   two bytes        [ 0xE0, 0xA0, 0x80 ],           // U+00000800   three bytes        [ 0xF0, 0x90, 0x80, 0x80 ],     // U+00010000   three bytes         // Last possible sequence of a certain length        [ 0x7F ],                       // U+0000007F   one byte        [ 0xDF, 0xBF ],                 // U+000007FF   two bytes        [ 0xEF, 0xBF, 0xBF ],           // U+0000FFFF   three bytes         // Other boundary conditions        [ 0xED, 0x9F, 0xBF ],        // U+0000D7FF   Last character before surrogates        [ 0xEE, 0x80, 0x80 ],        // U+0000E000   First character after surrogates        [ 0xEF, 0xBF, 0xBD ],        // U+0000FFFD   Unicode replacement character        [ 0xF4, 0x8F, 0xBF, 0xBF ],        // U+0010FFFF   Very last character         // Non-character code points        /*  NOTE: These are legal in UTF, and may be converted from            one UTF to another, however they do not represent Unicode            characters. These code points have been reserved by            Unicode as non-character code points. They are permissible            for data exchange within an application, but they are are            not permitted to be used as characters. Since this module            deals with UTF, and not with Unicode per se, we choose to            accept them here. */        [ 0xDF, 0xBE ],                 // U+0000FFFE        [ 0xDF, 0xBF ],                 // U+0000FFFF    ];     static ubyte[][] invalidStrings =    [        // First possible sequence of a certain length, but greater        // than U+10FFFF        [ 0xF8, 0x88, 0x80, 0x80, 0x80 ],           // U+00200000   five bytes        [ 0xFC, 0x84, 0x80, 0x80, 0x80, 0x80 ],     // U+04000000   six bytes         // Last possible sequence of a certain length, but greater than U+10FFFF        [ 0xF7, 0xBF, 0xBF, 0xBF ],                 // U+001FFFFF   four bytes        [ 0xFB, 0xBF, 0xBF, 0xBF, 0xBF ],           // U+03FFFFFF   five bytes        [ 0xFD, 0xBF, 0xBF, 0xBF, 0xBF, 0xBF ],     // U+7FFFFFFF   six bytes         // Other boundary conditions        [ 0xF4, 0x90, 0x80, 0x80 ],                 // U+00110000                                                    // First code                                                    // point after                                                    // last character         // Unexpected continuation bytes        [ 0x80 ],        [ 0xBF ],        [ 0x20, 0x80, 0x20 ],        [ 0x20, 0xBF, 0x20 ],        [ 0x80, 0x9F, 0xA0 ],         // Lonely start bytes        [ 0xC0 ],        [ 0xCF ],        [ 0x20, 0xC0, 0x20 ],        [ 0x20, 0xCF, 0x20 ],        [ 0xD0 ],        [ 0xDF ],        [ 0x20, 0xD0, 0x20 ],        [ 0x20, 0xDF, 0x20 ],        [ 0xE0 ],        [ 0xEF ],        [ 0x20, 0xE0, 0x20 ],        [ 0x20, 0xEF, 0x20 ],        [ 0xF0 ],        [ 0xF1 ],        [ 0xF2 ],        [ 0xF3 ],        [ 0xF4 ],        [ 0xF5 ],   // If this were legal it would start a character > U+10FFFF        [ 0xF6 ],   // If this were legal it would start a character > U+10FFFF        [ 0xF7 ],   // If this were legal it would start a character > U+10FFFF         [ 0xEF, 0xBF ],             // Three byte sequence with third byte missing        [ 0xF7, 0xBF, 0xBF ],       // Four byte sequence with fourth byte missing        [ 0xEF, 0xBF, 0xF7, 0xBF, 0xBF ],   // Concatenation of the above         // Impossible bytes        [ 0xF8 ],        [ 0xF9 ],        [ 0xFA ],        [ 0xFB ],        [ 0xFC ],        [ 0xFD ],        [ 0xFE ],        [ 0xFF ],        [ 0x20, 0xF8, 0x20 ],        [ 0x20, 0xF9, 0x20 ],        [ 0x20, 0xFA, 0x20 ],        [ 0x20, 0xFB, 0x20 ],        [ 0x20, 0xFC, 0x20 ],        [ 0x20, 0xFD, 0x20 ],        [ 0x20, 0xFE, 0x20 ],        [ 0x20, 0xFF, 0x20 ],         // Overlong sequences, all representing U+002F        /*  With a safe UTF-8 decoder, all of the following five overlong            representations of the ASCII character slash ("/") should be            rejected like a malformed UTF-8 sequence */        [ 0xC0, 0xAF ],        [ 0xE0, 0x80, 0xAF ],        [ 0xF0, 0x80, 0x80, 0xAF ],        [ 0xF8, 0x80, 0x80, 0x80, 0xAF ],        [ 0xFC, 0x80, 0x80, 0x80, 0x80, 0xAF ],         // Maximum overlong sequences        /*  Below you see the highest Unicode value that is still resulting in            an overlong sequence if represented with the given number of bytes.            This is a boundary test for safe UTF-8 decoders. All five            characters should be rejected like malformed UTF-8 sequences. */        [ 0xC1, 0xBF ],                             // U+0000007F        [ 0xE0, 0x9F, 0xBF ],                       // U+000007FF        [ 0xF0, 0x8F, 0xBF, 0xBF ],                 // U+0000FFFF        [ 0xF8, 0x87, 0xBF, 0xBF, 0xBF ],           // U+001FFFFF        [ 0xFC, 0x83, 0xBF, 0xBF, 0xBF, 0xBF ],     // U+03FFFFFF         // Overlong representation of the NUL character        /*  The following five sequences should also be rejected like malformed            UTF-8 sequences and should not be treated like the ASCII NUL            character. */        [ 0xC0, 0x80 ],        [ 0xE0, 0x80, 0x80 ],        [ 0xF0, 0x80, 0x80, 0x80 ],        [ 0xF8, 0x80, 0x80, 0x80, 0x80 ],        [ 0xFC, 0x80, 0x80, 0x80, 0x80, 0x80 ],         // Illegal code positions        /*  The following UTF-8 sequences should be rejected like malformed            sequences, because they never represent valid ISO 10646 characters            and a UTF-8 decoder that accepts them might introduce security            problems comparable to overlong UTF-8 sequences. */        [ 0xED, 0xA0, 0x80 ],       // U+D800        [ 0xED, 0xAD, 0xBF ],       // U+DB7F        [ 0xED, 0xAE, 0x80 ],       // U+DB80        [ 0xED, 0xAF, 0xBF ],       // U+DBFF        [ 0xED, 0xB0, 0x80 ],       // U+DC00        [ 0xED, 0xBE, 0x80 ],       // U+DF80        [ 0xED, 0xBF, 0xBF ],       // U+DFFF    ];     static string[] sanitizedStrings =    [        "\uFFFD","\uFFFD",        "\uFFFD","\uFFFD","\uFFFD","\uFFFD","\uFFFD","\uFFFD"," \uFFFD ",        " \uFFFD ","\uFFFD\uFFFD\uFFFD","\uFFFD","\uFFFD"," \uFFFD "," \uFFFD ",        "\uFFFD","\uFFFD"," \uFFFD "," \uFFFD ","\uFFFD","\uFFFD"," \uFFFD ",        " \uFFFD ","\uFFFD","\uFFFD","\uFFFD","\uFFFD","\uFFFD","\uFFFD",        "\uFFFD","\uFFFD","\uFFFD","\uFFFD","\uFFFD\uFFFD","\uFFFD","\uFFFD",        "\uFFFD","\uFFFD","\uFFFD","\uFFFD","\uFFFD","\uFFFD"," \uFFFD ",        " \uFFFD "," \uFFFD "," \uFFFD "," \uFFFD "," \uFFFD "," \uFFFD ",        " \uFFFD ","\uFFFD","\uFFFD","\uFFFD","\uFFFD","\uFFFD","\uFFFD",        "\uFFFD","\uFFFD","\uFFFD","\uFFFD","\uFFFD","\uFFFD","\uFFFD","\uFFFD",        "\uFFFD","\uFFFD","\uFFFD","\uFFFD","\uFFFD","\uFFFD","\uFFFD","\uFFFD",    ];     // HELPER FUNCTIONS    // we can probably do this better...    static char toHexDigit(int n)    {        return "0123456789ABCDEF"[n & 0xF];    }     static string makeReadable(string s)    {        string r = "\"";        foreach (char c;s)        {            if (c >= 0x20 && c < 0x80)            {                r ~= c;            }            else            {                r ~= "\\x";                r ~= toHexDigit(c >> 4);                r ~= toHexDigit(c);            }        }        r ~= "\"";        return r;    }     void transcodeReverse(Src,Dst)(immutable(Src)[] s, out immutable(Dst)[] r)    {        static if (is(Src == Dst))        {            return s;        }        else static if (is(Src == AsciiChar))        {            transcodeReverse!(char,Dst)(cast(string) s,r);        }        else        {            foreach_reverse (d;codePoints(s))            {                foreach_reverse (c;codeUnits!(Dst)(d))                {                    r = c ~ r;                }            }        }    }     // Make sure everything that should be valid, is    foreach (a;validStrings)    {        string s = cast(string) a;        assert(isValid(s),"Failed to validate: "~makeReadable(s));    }     // Make sure everything that shouldn't be valid, isn't    foreach (a;invalidStrings)    {        string s = cast(string) a;        assert(!isValid(s),"Incorrectly validated: "~makeReadable(s));    }     // Make sure we can sanitize everything bad    assert(invalidStrings.length == sanitizedStrings.length);    for (int i=0; i<invalidStrings.length; ++i)    {        string s = cast(string) invalidStrings[i];        string t = sanitize(s);        assert(isValid(t));        assert(t == sanitizedStrings[i]);        ubyte[] u = cast(ubyte[]) t;        validStrings ~= u;    }     // Make sure all transcodings work in both directions, using both forward    // and reverse iteration    foreach (a; validStrings)    {        string s = cast(string) a;        string s2;        wstring ws, ws2;        dstring ds, ds2;         transcode(s,ws);        assert(isValid(ws));        transcode(ws,s2);        assert(s == s2);         transcode(s,ds);        assert(isValid(ds));        transcode(ds,s2);        assert(s == s2);         transcode(ws,s);        assert(isValid(s));        transcode(s,ws2);        assert(ws == ws2);         transcode(ws,ds);        assert(isValid(ds));        transcode(ds,ws2);        assert(ws == ws2);         transcode(ds,s);        assert(isValid(s));        transcode(s,ds2);        assert(ds == ds2);         transcode(ds,ws);        assert(isValid(ws));        transcode(ws,ds2);        assert(ds == ds2);         transcodeReverse(s,ws);        assert(isValid(ws));        transcodeReverse(ws,s2);        assert(s == s2);         transcodeReverse(s,ds);        assert(isValid(ds));        transcodeReverse(ds,s2);        assert(s == s2);         transcodeReverse(ws,s);        assert(isValid(s));        transcodeReverse(s,ws2);        assert(ws == ws2);         transcodeReverse(ws,ds);        assert(isValid(ds));        transcodeReverse(ds,ws2);        assert(ws == ws2);         transcodeReverse(ds,s);        assert(isValid(s));        transcodeReverse(s,ds2);        assert(ds == ds2);         transcodeReverse(ds,ws);        assert(isValid(ws));        transcodeReverse(ws,ds2);        assert(ds == ds2);    }     // Make sure the non-UTF encodings work too    {        auto s = "\u20AC100";        Windows1252String t;        transcode(s,t);        assert(t == cast(Windows1252Char[])[0x80, '1', '0', '0']);        string u;        transcode(s,u);        assert(s == u);        Latin1String v;        transcode(s,v);        assert(cast(string) v == "?100");        AsciiString w;        transcode(v,w);        assert(cast(string) w == "?100");        s = "\u017Dlu\u0165ou\u010Dk\u00FD k\u016F\u0148";        Latin2String x;        transcode(s,x);        assert(x == cast(Latin2Char[])[0xae, 'l', 'u', 0xbb, 'o', 'u', 0xe8, 'k', 0xfd, ' ', 'k', 0xf9, 0xf2]);        Windows1250String y;        transcode(s,y);        assert(y == cast(Windows1250Char[])[0x8e, 'l', 'u', 0x9d, 'o', 'u', 0xe8, 'k', 0xfd, ' ', 'k', 0xf9, 0xf2]);        s = "\u0402lu\u0403ou\u201D\u045C k\u0414\u044F";        Windows1251String s51;        transcode(s,s51);        assert(s51 == cast(Windows1251Char[])[0x80, 'l', 'u', 0x81, 'o', 'u', 0x94, 0x9d, ' ', 'k', 0xc4, 0xff]);    }     // Make sure we can count properly    {        assert(encodedLength!(char)('A') == 1);        assert(encodedLength!(char)('\u00E3') == 2);        assert(encodedLength!(char)('\u2028') == 3);        assert(encodedLength!(char)('\U0010FFF0') == 4);        assert(encodedLength!(wchar)('A') == 1);        assert(encodedLength!(wchar)('\U0010FFF0') == 2);    }     // Make sure we can write into mutable arrays    {        char[4] buffer;        auto n = encode(cast(dchar)'\u00E3',buffer);        assert(n == 2);        assert(buffer[0] == 0xC3);        assert(buffer[1] == 0xA3);    }} //============================================================================= /** Special value returned by `safeDecode` */enum dchar INVALID_SEQUENCE = cast(dchar) 0xFFFFFFFF; template EncoderFunctions(){    // Various forms of read     template ReadFromString()    {        @property bool canRead() { return s.length != 0; }        E peek() @safe pure @nogc nothrow { return s[0]; }        E read() @safe pure @nogc nothrow { E t = s[0]; s = s[1..$]; return t; }    }     template ReverseReadFromString()    {        @property bool canRead() { return s.length != 0; }        E peek() @safe pure @nogc nothrow { return s[$-1]; }        E read() @safe pure @nogc nothrow { E t = s[$-1]; s = s[0..$-1]; return t; }    } 

Lines 1–500 of 3854. Load more