// Written in the D programming language. /++    Encode and decode UTF-8, UTF-16 and UTF-32 strings.     UTF character support is restricted to    $(D '\u0000' &lt;= character &lt;= '\U0010FFFF'). $(SCRIPT inhibitQuickIndex = 1;)$(DIVC quickindex,$(BOOKTABLE,$(TR $(TH Category) $(TH Functions))$(TR $(TD Decode) $(TD    $(LREF decode)    $(LREF decodeFront)))$(TR $(TD Lazy decode) $(TD    $(LREF byCodeUnit)    $(LREF byChar)    $(LREF byWchar)    $(LREF byDchar)    $(LREF byUTF)))$(TR $(TD Encode) $(TD    $(LREF encode)    $(LREF toUTF8)    $(LREF toUTF16)    $(LREF toUTF32)    $(LREF toUTFz)    $(LREF toUTF16z)))$(TR $(TD Length) $(TD    $(LREF codeLength)    $(LREF count)    $(LREF stride)    $(LREF strideBack)))$(TR $(TD Index) $(TD    $(LREF toUCSindex)    $(LREF toUTFindex)))$(TR $(TD Validation) $(TD    $(LREF isValidDchar)    $(LREF isValidCodepoint)    $(LREF validate)))$(TR $(TD Miscellaneous) $(TD    $(LREF replacementDchar)    $(LREF UseReplacementDchar)    $(LREF UTFException)))))    See_Also:        $(LINK2 http://en.wikipedia.org/wiki/Unicode, Wikipedia)<br>        $(LINK http://www.cl.cam.ac.uk/~mgk25/unicode.html#utf-8)<br>        $(LINK https://web.archive.org/web/20100113043530/https://anubis.dkuug.dk/JTC1/SC2/WG2/docs/n1335)    Copyright: Copyright The D Language Foundation 2000 - 2012.    License:   $(HTTP www.boost.org/LICENSE_1_0.txt, Boost License 1.0).    Authors:   $(HTTP digitalmars.com, Walter Bright) and               $(HTTP jmdavisprog.com, Jonathan M Davis)    Source:    $(PHOBOSSRC std/utf.d)   +/module std.utf; import std.exception : basicExceptionCtors;import core.exception : UnicodeException;import std.meta : AliasSeq;import std.range;import std.traits : isAutodecodableString, isConvertibleToString,    isSomeChar, isSomeString, isStaticArray, Unqual;import std.typecons : Flag, Yes, No;  /++    Exception thrown on errors in std.utf functions.  +/class UTFException : UnicodeException{    import core.internal.string : unsignedToTempString, UnsignedStringBuf;     uint[4] sequence;    size_t  len;     @safe pure nothrow @nogc    UTFException setSequence(scope uint[] data...) return    {        assert(data.length <= 4);         len = data.length < 4 ? data.length : 4;        sequence[0 .. len] = data[0 .. len];         return this;    }     // FIXME: Use std.exception.basicExceptionCtors here once    // https://issues.dlang.org/show_bug.cgi?id=11500 is fixed     /**    Standard exception constructors.     */    this(string msg, string file = __FILE__, size_t line = __LINE__,         Throwable next = null) @nogc @safe pure nothrow    {        super(msg, 0, file, line, next);    }    /// ditto    this(string msg, size_t index, string file = __FILE__,         size_t line = __LINE__, Throwable next = null) @safe pure nothrow    {        UnsignedStringBuf buf = void;        msg ~= " (at index " ~ unsignedToTempString(index, buf) ~ ")";        super(msg, index, file, line, next);    }     /**    Returns:        A `string` detailing the invalid UTF sequence.     */    override string toString() const    {        if (len == 0)        {            /* Exception.toString() is not marked as const, although             * it is const-compatible.             */            //return super.toString();            auto e = () @trusted { return cast(Exception) super; } ();            return e.toString();        }         string result = "Invalid UTF sequence:";         foreach (i; sequence[0 .. len])        {            UnsignedStringBuf buf = void;            result ~= ' ';            auto h = unsignedToTempString!16(i, buf);            if (h.length == 1)                result ~= '0';            result ~= h;            result ~= 'x';        }         if (super.msg.length > 0)        {            result ~= " - ";            result ~= super.msg;        }         return result;    }} ///@safe unittest{    import std.exception : assertThrown;     char[4] buf;    assertThrown!UTFException(encode(buf, cast(dchar) 0xD800));    assertThrown!UTFException(encode(buf, cast(dchar) 0xDBFF));    assertThrown!UTFException(encode(buf, cast(dchar) 0xDC00));    assertThrown!UTFException(encode(buf, cast(dchar) 0xDFFF));    assertThrown!UTFException(encode(buf, cast(dchar) 0x110000));} /*   Provide array of invalidly encoded UTF strings. Useful for testing.    Params:        Char = char, wchar, or dchar    Returns:        an array of invalidly encoded UTF strings */ package auto invalidUTFstrings(Char)() @safe pure @nogc nothrowif (isSomeChar!Char){    static if (is(Char == char))    {        enum x = 0xDC00;         // invalid surrogate value        enum y = 0x110000;       // out of range         static immutable string[8] result =        [            "\x80",             // not a start byte            "\xC0",             // truncated            "\xC0\xC0",         // invalid continuation            "\xF0\x82\x82\xAC", // overlong            [              0xE0 | (x >> 12),              0x80 | ((x >> 6) & 0x3F),              0x80 | (x & 0x3F)            ],            [              cast(char)(0xF0 | (y >> 18)),              cast(char)(0x80 | ((y >> 12) & 0x3F)),              cast(char)(0x80 | ((y >> 6) & 0x3F)),              cast(char)(0x80 | (y & 0x3F))            ],            [              cast(char)(0xF8 | 3),     // 5 byte encoding              cast(char)(0x80 | 3),              cast(char)(0x80 | 3),              cast(char)(0x80 | 3),              cast(char)(0x80 | 3),            ],            [              cast(char)(0xFC | 3),     // 6 byte encoding              cast(char)(0x80 | 3),              cast(char)(0x80 | 3),              cast(char)(0x80 | 3),              cast(char)(0x80 | 3),              cast(char)(0x80 | 3),            ],        ];         return result[];    }    else static if (is(Char == wchar))    {        static immutable wstring[5] result =        [            [              cast(wchar) 0xDC00,            ],            [              cast(wchar) 0xDFFF,            ],            [              cast(wchar) 0xDBFF,              cast(wchar) 0xDBFF,            ],            [              cast(wchar) 0xDBFF,              cast(wchar) 0xE000,            ],            [              cast(wchar) 0xD800,            ],        ];         return result[];    }    else static if (is(Char == dchar))    {        static immutable dstring[3] result =        [            [ cast(dchar) 0x110000 ],            [ cast(dchar) 0x00D800 ],            [ cast(dchar) 0x00DFFF ],        ];         return result;    }    else        static assert(0);} /++    Check whether the given Unicode code point is valid.     Params:        c = code point to check     Returns:        `true` if and only if `c` is a valid Unicode code point     Note:    `'\uFFFE'` and `'\uFFFF'` are considered valid by `isValidDchar`,    as they are permitted for internal use by an application, but they are    not allowed for interchange by the Unicode standard.  +/bool isValidDchar(dchar c) pure nothrow @safe @nogc{    return c < 0xD800 || (c > 0xDFFF && c <= 0x10FFFF);} ///@safe @nogc pure nothrow unittest{    assert( isValidDchar(cast(dchar) 0x41));    assert( isValidDchar(cast(dchar) 0x00));    assert(!isValidDchar(cast(dchar) 0xD800));    assert(!isValidDchar(cast(dchar) 0x11FFFF));} pure nothrow @safe @nogc unittest{    import std.exception;     assertCTFEable!(    {    assert( isValidDchar(cast(dchar)'a') == true);    assert( isValidDchar(cast(dchar) 0x1FFFFF) == false);     assert(!isValidDchar(cast(dchar) 0x00D800));    assert(!isValidDchar(cast(dchar) 0x00DBFF));    assert(!isValidDchar(cast(dchar) 0x00DC00));    assert(!isValidDchar(cast(dchar) 0x00DFFF));    assert( isValidDchar(cast(dchar) 0x00FFFE));    assert( isValidDchar(cast(dchar) 0x00FFFF));    assert( isValidDchar(cast(dchar) 0x01FFFF));    assert( isValidDchar(cast(dchar) 0x10FFFF));    assert(!isValidDchar(cast(dchar) 0x110000));    });} /**Checks if a single character forms a valid code point. When standing alone, some characters are invalid code points. Forexample the `wchar` `0xD800` is a so called high surrogate, which canonly be interpreted together with a low surrogate following it. As astandalone character it is considered invalid. See $(LINK2 http://www.unicode.org/versions/Unicode13.0.0/,Unicode Standard, D90, D91 and D92) for more details. Params:    c = character to test    Char = character type of `c` Returns:    `true`, if `c` forms a valid code point. */bool isValidCodepoint(Char)(Char c)if (isSomeChar!Char){    alias UChar = typeof(cast() c);    static if (is(UChar == char))    {        return c <= 0x7F;    }    else static if (is(UChar == wchar))    {        return c <= 0xD7FF || c >= 0xE000;    }    else static if (is(UChar == dchar))    {        return isValidDchar(c);    }    else        static assert(false, "unknown character type: `" ~ Char.stringof ~ "`");} ///@safe pure nothrow unittest{    assert( isValidCodepoint(cast(char) 0x40));    assert(!isValidCodepoint(cast(char) 0x80));    assert( isValidCodepoint(cast(wchar) 0x1234));    assert(!isValidCodepoint(cast(wchar) 0xD800));    assert( isValidCodepoint(cast(dchar) 0x0010FFFF));    assert(!isValidCodepoint(cast(dchar) 0x12345678));} /++    Calculate the length of the UTF sequence starting at `index`    in `str`.     Params:        str = $(REF_ALTTEXT input range, isInputRange, std,range,primitives)        of UTF code units. Must be random access if `index` is passed        index = starting index of UTF sequence (default: `0`)     Returns:        The number of code units in the UTF sequence. For UTF-8, this is a        value between 1 and 4 (as per $(HTTP tools.ietf.org/html/rfc3629#section-3, RFC 3629$(COMMA) section 3)).        For UTF-16, it is either 1 or 2. For UTF-32, it is always 1.     Throws:        May throw a `UTFException` if `str[index]` is not the start of a        valid UTF sequence.     Note:        `stride` will only analyze the first `str[index]` element. It        will not fully verify the validity of the UTF sequence, nor even verify        the presence of the sequence: it will not actually guarantee that        $(D index + stride(str, index) <= str.length).  +/uint stride(S)(auto ref S str, size_t index)if (is(S : const char[]) ||    (isRandomAccessRange!S && is(immutable ElementType!S == immutable char))){    static if (is(typeof(str.length) : ulong))        assert(index < str.length, "Past the end of the UTF-8 sequence");    immutable c = str[index];     if (c < 0x80)        return 1;    else        return strideImpl(c, index);} /// Dittouint stride(S)(auto ref S str)if (is(S : const char[]) ||    (isInputRange!S && is(immutable ElementType!S == immutable char))){    static if (is(S : const char[]))        immutable c = str[0];    else        immutable c = str.front;     if (c < 0x80)        return 1;    else        return strideImpl(c, 0);} @system unittest{    import core.exception : AssertError;    import std.conv : to;    import std.exception;    import std.string : format;    import std.traits : FunctionAttribute, functionAttributes, isSafe;    static void test(string s, dchar c, size_t i = 0, size_t line = __LINE__)    {        enforce(stride(s, i) == codeLength!char(c),                new AssertError(format("Unit test failure string: %s", s), __FILE__, line));         enforce(stride(RandomCU!char(s), i) == codeLength!char(c),                new AssertError(format("Unit test failure range: %s", s), __FILE__, line));         auto refRandom = new RefRandomCU!char(s);        immutable randLen = refRandom.length;        enforce(stride(refRandom, i) == codeLength!char(c),                new AssertError(format("Unit test failure rand ref range: %s", s), __FILE__, line));        enforce(refRandom.length == randLen,                new AssertError(format("Unit test failure rand ref range length: %s", s), __FILE__, line));         if (i == 0)        {            enforce(stride(s) == codeLength!char(c),                    new AssertError(format("Unit test failure string 0: %s", s), __FILE__, line));             enforce(stride(InputCU!char(s)) == codeLength!char(c),                    new AssertError(format("Unit test failure range 0: %s", s), __FILE__, line));             auto refBidir = new RefBidirCU!char(s);            immutable bidirLen = refBidir.length;            enforce(stride(refBidir) == codeLength!char(c),                    new AssertError(format("Unit test failure bidir ref range code length: %s", s), __FILE__, line));            enforce(refBidir.length == bidirLen,                    new AssertError(format("Unit test failure bidir ref range length: %s", s), __FILE__, line));        }    }     assertCTFEable!(    {    test("a", 'a');    test(" ", ' ');    test("\u2029", '\u2029'); //paraSep    test("\u0100", '\u0100');    test("\u0430", '\u0430');    test("\U00010143", '\U00010143');    test("abcdefcdef", 'a');    test("hello\U00010143\u0100\U00010143", 'h', 0);    test("hello\U00010143\u0100\U00010143", 'e', 1);    test("hello\U00010143\u0100\U00010143", 'l', 2);    test("hello\U00010143\u0100\U00010143", 'l', 3);    test("hello\U00010143\u0100\U00010143", 'o', 4);    test("hello\U00010143\u0100\U00010143", '\U00010143', 5);    test("hello\U00010143\u0100\U00010143", '\u0100', 9);    test("hello\U00010143\u0100\U00010143", '\U00010143', 11);     foreach (S; AliasSeq!(char[], const char[], string))    {        enum str = to!S("hello world");        static assert(isSafe!({ stride(str, 0); }));        static assert(isSafe!({ stride(str);    }));        static assert((functionAttributes!({ stride(str, 0); }) & FunctionAttribute.pure_) != 0);        static assert((functionAttributes!({ stride(str);    }) & FunctionAttribute.pure_) != 0);    }    });} @safe unittest // invalid start bytes{    import std.exception : assertThrown;    immutable char[] invalidStartBytes = [        0b1111_1000, // indicating a sequence length of 5        0b1111_1100, // 6        0b1111_1110, // 7        0b1111_1111, // 8        0b1000_0000, // continuation byte    ];    foreach (c; invalidStartBytes)        assertThrown!UTFException(stride([c]));} /// Dittouint stride(S)(auto ref S str, size_t index)if (is(S : const wchar[]) ||    (isRandomAccessRange!S && is(immutable ElementType!S == immutable wchar))){    static if (is(typeof(str.length) : ulong))

Lines 1–500 of 4731. Load more