/* * Unicode utilities *  * Copyright (c) 2017-2018 Fabrice Bellard * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. */#include <stdlib.h>#include <stdio.h>#include <stdarg.h>#include <string.h>#include <assert.h> #include "cutils.h"#include "libunicode.h"#include "libunicode-table.h" enum {    RUN_TYPE_U,    RUN_TYPE_L,    RUN_TYPE_UF,    RUN_TYPE_LF,    RUN_TYPE_UL,    RUN_TYPE_LSU,    RUN_TYPE_U2L_399_EXT2,    RUN_TYPE_UF_D20,    RUN_TYPE_UF_D1_EXT,    RUN_TYPE_U_EXT,    RUN_TYPE_LF_EXT,    RUN_TYPE_U_EXT2,    RUN_TYPE_L_EXT2,    RUN_TYPE_U_EXT3,}; /* conv_type:   0 = to upper    1 = to lower   2 = case folding (= to lower with modifications) */int lre_case_conv(uint32_t *res, uint32_t c, int conv_type){    if (c < 128) {        if (conv_type) {            if (c >= 'A' && c <= 'Z') {                c = c - 'A' + 'a';            }        } else {            if (c >= 'a' && c <= 'z') {                c = c - 'a' + 'A';            }        }    } else {        uint32_t v, code, data, type, len, a, is_lower;        int idx, idx_min, idx_max;                is_lower = (conv_type != 0);        idx_min = 0;        idx_max = countof(case_conv_table1) - 1;        while (idx_min <= idx_max) {            idx = (unsigned)(idx_max + idx_min) / 2;            v = case_conv_table1[idx];            code = v >> (32 - 17);            len = (v >> (32 - 17 - 7)) & 0x7f;            if (c < code) {                idx_max = idx - 1;            } else if (c >= code + len) {                idx_min = idx + 1;            } else {                type = (v >> (32 - 17 - 7 - 4)) & 0xf;                data = ((v & 0xf) << 8) | case_conv_table2[idx];                switch(type) {                case RUN_TYPE_U:                case RUN_TYPE_L:                case RUN_TYPE_UF:                case RUN_TYPE_LF:                    if (conv_type == (type & 1) ||                        (type >= RUN_TYPE_UF && conv_type == 2)) {                        c = c - code + (case_conv_table1[data] >> (32 - 17));                    }                    break;                case RUN_TYPE_UL:                    a = c - code;                    if ((a & 1) != (1 - is_lower))                        break;                    c = (a ^ 1) + code;                    break;                case RUN_TYPE_LSU:                    a = c - code;                    if (a == 1) {                        c += 2 * is_lower - 1;                    } else if (a == (1 - is_lower) * 2) {                        c += (2 * is_lower - 1) * 2;                    }                    break;                case RUN_TYPE_U2L_399_EXT2:                    if (!is_lower) {                        res[0] = c - code + case_conv_ext[data >> 6];                        res[1] = 0x399;                        return 2;                    } else {                        c = c - code + case_conv_ext[data & 0x3f];                    }                    break;                case RUN_TYPE_UF_D20:                    if (conv_type == 1)                        break;                    c = data + (conv_type == 2) * 0x20;                    break;                case RUN_TYPE_UF_D1_EXT:                    if (conv_type == 1)                        break;                    c = case_conv_ext[data] + (conv_type == 2);                    break;                case RUN_TYPE_U_EXT:                case RUN_TYPE_LF_EXT:                    if (is_lower != (type - RUN_TYPE_U_EXT))                        break;                    c = case_conv_ext[data];                    break;                case RUN_TYPE_U_EXT2:                case RUN_TYPE_L_EXT2:                    if (conv_type != (type - RUN_TYPE_U_EXT2))                        break;                    res[0] = c - code + case_conv_ext[data >> 6];                    res[1] = case_conv_ext[data & 0x3f];                    return 2;                default:                case RUN_TYPE_U_EXT3:                    if (conv_type != 0)                        break;                    res[0] = case_conv_ext[data >> 8];                    res[1] = case_conv_ext[(data >> 4) & 0xf];                    res[2] = case_conv_ext[data & 0xf];                    return 3;                }                break;            }        }    }    res[0] = c;    return 1;} static uint32_t get_le24(const uint8_t *ptr){#if defined(__x86__) || defined(__x86_64__)    return *(uint16_t *)ptr | (ptr[2] << 16);#else    return ptr[0] | (ptr[1] << 8) | (ptr[2] << 16);#endif} #define UNICODE_INDEX_BLOCK_LEN 32 /* return -1 if not in table, otherwise the offset in the block */static int get_index_pos(uint32_t *pcode, uint32_t c,                         const uint8_t *index_table, int index_table_len){    uint32_t code, v;    int idx_min, idx_max, idx;     idx_min = 0;    v = get_le24(index_table);    code = v & ((1 << 21) - 1);    if (c < code) {        *pcode = 0;        return 0;    }    idx_max = index_table_len - 1;    code = get_le24(index_table + idx_max * 3);    if (c >= code)        return -1;    /* invariant: tab[idx_min] <= c < tab2[idx_max] */    while ((idx_max - idx_min) > 1) {        idx = (idx_max + idx_min) / 2;        v = get_le24(index_table + idx * 3);        code = v & ((1 << 21) - 1);        if (c < code) {            idx_max = idx;        } else {            idx_min = idx;        }    }    v = get_le24(index_table + idx_min * 3);    *pcode = v & ((1 << 21) - 1);    return (idx_min + 1) * UNICODE_INDEX_BLOCK_LEN + (v >> 21);} static BOOL lre_is_in_table(uint32_t c, const uint8_t *table,                            const uint8_t *index_table, int index_table_len){    uint32_t code, b, bit;    int pos;    const uint8_t *p;        pos = get_index_pos(&code, c, index_table, index_table_len);    if (pos < 0)        return FALSE; /* outside the table */    p = table + pos;    bit = 0;    for(;;) {        b = *p++;        if (b < 64) {            code += (b >> 3) + 1;            if (c < code)                return bit;            bit ^= 1;            code += (b & 7) + 1;        } else if (b >= 0x80) {            code += b - 0x80 + 1;        } else if (b < 0x60) {            code += (((b - 0x40) << 8) | p[0]) + 1;            p++;        } else {            code += (((b - 0x60) << 16) | (p[0] << 8) | p[1]) + 1;            p += 2;        }        if (c < code)            return bit;        bit ^= 1;    }} BOOL lre_is_cased(uint32_t c){    uint32_t v, code, len;    int idx, idx_min, idx_max;            idx_min = 0;    idx_max = countof(case_conv_table1) - 1;    while (idx_min <= idx_max) {        idx = (unsigned)(idx_max + idx_min) / 2;        v = case_conv_table1[idx];        code = v >> (32 - 17);        len = (v >> (32 - 17 - 7)) & 0x7f;        if (c < code) {            idx_max = idx - 1;        } else if (c >= code + len) {            idx_min = idx + 1;        } else {            return TRUE;        }    }    return lre_is_in_table(c, unicode_prop_Cased1_table,                           unicode_prop_Cased1_index,                           sizeof(unicode_prop_Cased1_index) / 3);} BOOL lre_is_case_ignorable(uint32_t c){    return lre_is_in_table(c, unicode_prop_Case_Ignorable_table,                           unicode_prop_Case_Ignorable_index,                           sizeof(unicode_prop_Case_Ignorable_index) / 3);} /* character range */ static __maybe_unused void cr_dump(CharRange *cr){    int i;    for(i = 0; i < cr->len; i++)        printf("%d: 0x%04x\n", i, cr->points[i]);} static void *cr_default_realloc(void *opaque, void *ptr, size_t size){    return realloc(ptr, size);} void cr_init(CharRange *cr, void *mem_opaque, DynBufReallocFunc *realloc_func){    cr->len = cr->size = 0;    cr->points = NULL;    cr->mem_opaque = mem_opaque;    cr->realloc_func = realloc_func ? realloc_func : cr_default_realloc;} void cr_free(CharRange *cr){    cr->realloc_func(cr->mem_opaque, cr->points, 0);} int cr_realloc(CharRange *cr, int size){    int new_size;    uint32_t *new_buf;        if (size > cr->size) {        new_size = max_int(size, cr->size * 3 / 2);        new_buf = cr->realloc_func(cr->mem_opaque, cr->points,                                   new_size * sizeof(cr->points[0]));        if (!new_buf)            return -1;        cr->points = new_buf;        cr->size = new_size;    }    return 0;} int cr_copy(CharRange *cr, const CharRange *cr1){    if (cr_realloc(cr, cr1->len))        return -1;    memcpy(cr->points, cr1->points, sizeof(cr->points[0]) * cr1->len);    cr->len = cr1->len;    return 0;} /* merge consecutive intervals and remove empty intervals */static void cr_compress(CharRange *cr){    int i, j, k, len;    uint32_t *pt;        pt = cr->points;    len = cr->len;    i = 0;    j = 0;    k = 0;    while ((i + 1) < len) {        if (pt[i] == pt[i + 1]) {            /* empty interval */            i += 2;        } else {            j = i;            while ((j + 3) < len && pt[j + 1] == pt[j + 2])                j += 2;            /* just copy */            pt[k] = pt[i];            pt[k + 1] = pt[j + 1];            k += 2;            i = j + 2;        }    }    cr->len = k;} /* union or intersection */int cr_op(CharRange *cr, const uint32_t *a_pt, int a_len,          const uint32_t *b_pt, int b_len, int op){    int a_idx, b_idx, is_in;    uint32_t v;        a_idx = 0;    b_idx = 0;    for(;;) {        /* get one more point from a or b in increasing order */        if (a_idx < a_len && b_idx < b_len) {            if (a_pt[a_idx] < b_pt[b_idx]) {                goto a_add;            } else if (a_pt[a_idx] == b_pt[b_idx]) {                v = a_pt[a_idx];                a_idx++;                b_idx++;            } else {                goto b_add;            }        } else if (a_idx < a_len) {        a_add:            v = a_pt[a_idx++];        } else if (b_idx < b_len) {        b_add:            v = b_pt[b_idx++];        } else {            break;        }        /* add the point if the in/out status changes */        switch(op) {        case CR_OP_UNION:            is_in = (a_idx & 1) | (b_idx & 1);            break;        case CR_OP_INTER:            is_in = (a_idx & 1) & (b_idx & 1);            break;        case CR_OP_XOR:            is_in = (a_idx & 1) ^ (b_idx & 1);            break;        default:            abort();        }        if (is_in != (cr->len & 1)) {            if (cr_add_point(cr, v))                return -1;        }    }    cr_compress(cr);    return 0;} int cr_union1(CharRange *cr, const uint32_t *b_pt, int b_len){    CharRange a = *cr;    int ret;    cr->len = 0;    cr->size = 0;    cr->points = NULL;    ret = cr_op(cr, a.points, a.len, b_pt, b_len, CR_OP_UNION);    cr_free(&a);    return ret;} int cr_invert(CharRange *cr){    int len;    len = cr->len;    if (cr_realloc(cr, len + 2))        return -1;    memmove(cr->points + 1, cr->points, len * sizeof(cr->points[0]));    cr->points[0] = 0;    cr->points[len + 1] = UINT32_MAX;    cr->len = len + 2;    cr_compress(cr);    return 0;} #ifdef CONFIG_ALL_UNICODE BOOL lre_is_id_start(uint32_t c){    return lre_is_in_table(c, unicode_prop_ID_Start_table,                           unicode_prop_ID_Start_index,                           sizeof(unicode_prop_ID_Start_index) / 3);} BOOL lre_is_id_continue(uint32_t c){    return lre_is_id_start(c) ||        lre_is_in_table(c, unicode_prop_ID_Continue1_table,                        unicode_prop_ID_Continue1_index,                        sizeof(unicode_prop_ID_Continue1_index) / 3);} #define UNICODE_DECOMP_LEN_MAX 18 typedef enum {    DECOMP_TYPE_C1, /* 16 bit char */    DECOMP_TYPE_L1, /* 16 bit char table */    DECOMP_TYPE_L2,    DECOMP_TYPE_L3,    DECOMP_TYPE_L4,    DECOMP_TYPE_L5, /* XXX: not used */    DECOMP_TYPE_L6, /* XXX: could remove */    DECOMP_TYPE_L7, /* XXX: could remove */    DECOMP_TYPE_LL1, /* 18 bit char table */    DECOMP_TYPE_LL2,    DECOMP_TYPE_S1, /* 8 bit char table */    DECOMP_TYPE_S2,    DECOMP_TYPE_S3,    DECOMP_TYPE_S4,    DECOMP_TYPE_S5,    DECOMP_TYPE_I1, /* increment 16 bit char value */    DECOMP_TYPE_I2_0,    DECOMP_TYPE_I2_1,    DECOMP_TYPE_I3_1,    DECOMP_TYPE_I3_2,    DECOMP_TYPE_I4_1,    DECOMP_TYPE_I4_2,    DECOMP_TYPE_B1, /* 16 bit base + 8 bit offset */    DECOMP_TYPE_B2,    DECOMP_TYPE_B3,    DECOMP_TYPE_B4,    DECOMP_TYPE_B5,    DECOMP_TYPE_B6,    DECOMP_TYPE_B7,    DECOMP_TYPE_B8,    DECOMP_TYPE_B18,    DECOMP_TYPE_LS2,    DECOMP_TYPE_PAT3,    DECOMP_TYPE_S2_UL,    DECOMP_TYPE_LS2_UL,} DecompTypeEnum; static uint32_t unicode_get_short_code(uint32_t c){    static const uint16_t unicode_short_table[2] = { 0x2044, 0x2215 };     if (c < 0x80)        return c;    else if (c < 0x80 + 0x50)        return c - 0x80 + 0x300;    else        return unicode_short_table[c - 0x80 - 0x50];}

Lines 1–500 of 1556. Load more