/*  Basic UTF-8 manipulation routines  by Jeff Bezanson  placed in the public domain Fall 2005   This code is designed to provide the utilities you need to manipulate  UTF-8 as an internal string encoding. These functions do not perform the  error checking normally needed when handling UTF-8 data, so if you happen  to be from the Unicode Consortium you will want to flay me alive.  I do this because error checking can be performed at the boundaries (I/O),  with these routines reserved for higher performance on data known to be  valid.  A UTF-8 validation routine is included.*/ #include "sl.h" static const u32int offsetsFromUTF8[6] = {	0x00000000U, 0x00003080U, 0x000E2080U,	0x03C82080U, 0xFA082080U, 0x82082080U}; static const char trailingBytesForUTF8[256] = {	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,	1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,	2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5}; /* returns length of next utf-8 sequence */usizeu8_seqlen(const char *s){	return trailingBytesForUTF8[(u8int)s[0]] + 1;} /* modified runenlen from 9front */usizeu8_runelen(const char *s, usize m, bool *err){	const char *es = s + m;	usize n; 	*err = false;	for(n = 0; s < es; n++) {		if(*(const u8int*)s < Runeself){			s++;			continue;		}		if(!fullrune(s, es-s)){			*err = true;			break;		}		Rune r;		s += chartorune(&r, s);		if(r == Runeerror){			*err = true;			break;		}	}	return n;} ssizeu8_strwidth(const char *s, usize n, bool *err){	ssize w = 0;	*err = false;	for(usize i = 0; i < n;){		if(!fullrune(s+i, n-i)){			*err = true;			break;		}		Rune r;		i += chartorune(&r, s+i);		if(r == Runeerror){			*err = true;			break;		}		w += sl_wcwidth(r);	}	return w;} /* next character without NUL character terminator */Runeu8_nextmemchar(const char *s, usize *i){	usize sz = trailingBytesForUTF8[(u8int)s[*i]];	Rune ch = 0;	for(usize j = 0; j <= sz; j++){		ch <<= 6;		ch += (u8int)s[(*i)++];	};	return ch - offsetsFromUTF8[sz];} booloctal_digit(char c){	return c >= '0' && c <= '7';} boolhex_digit(char c){	return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');} charread_escape_control_char(char c){	switch(c){	case 'n': return '\n';	case 't': return '\t';	case 'a': return '\a';	case 'b': return '\b';	case 'e': return 0x1b;	case 'f': return '\f';	case 'r': return '\r';	case 'v': return '\v';	}	return c;} static inline intbuf_put2c(char *buf, const char *src){	buf[0] = src[0];	buf[1] = src[1];	buf[2] = '\0';	return 2;} intu8_escape_rune(char *buf, usize sz, Rune ch){	assert(sz > 12);	if(ch == '\\')		return buf_put2c(buf, "\\\\");	if(ch >= 0x20 && ch < 0x7f){		buf[0] = ch;		buf[1] = '\0';		return 1;	}	if(ch > 0xffff)		return snprintf(buf, sz, "\\U%.8"PRIx32, ch);	if(ch >= Runeself)		return snprintf(buf, sz, "\\u%04"PRIx32, ch);	switch(ch){	case '\n': return buf_put2c(buf, "\\n");	case '\t': return buf_put2c(buf, "\\t");	case '\a': return buf_put2c(buf, "\\a");	case '\b': return buf_put2c(buf, "\\b");	case 0x1b: return buf_put2c(buf, "\\e");	case '\f': return buf_put2c(buf, "\\f");	case '\r': return buf_put2c(buf, "\\r");	case '\v': return buf_put2c(buf, "\\v");	}	return snprintf(buf, sz, "\\x%02"PRIx32, ch);} usizeu8_escape(char *buf, usize sz, const char *src, usize *pi, usize end, bool escape_quotes, bool ascii){	usize i = *pi, i0;	Rune ch;	char *start = buf;	char *blim = start + sz-12;	assert(sz > 12); 	while(i < end && buf < blim){		// sz-12: leaves room for longest escape sequence		if(escape_quotes && src[i] == '"'){			buf += buf_put2c(buf, "\\\"");			i++;		}else if(src[i] == '\\'){			buf += buf_put2c(buf, "\\\\");			i++;		}else{			i0 = i;			ch = u8_nextmemchar(src, &i);			if(ascii || !sl_iswprint(ch)){				buf += u8_escape_rune(buf, sz - (buf-start), ch);			}else{				i = i0;				do{					*buf++ = src[i++];				}while(!isutf(src[i]));			}		}	}	*buf++ = '\0';	*pi = i;	return buf - start;} u8int*u8_memchr(u8int *s, Rune ch, usize sz, usize *charn){	usize i = 0, lasti = 0;	Rune c;	int csz; 	*charn = 0;	while(i < sz){		c = csz = 0;		do{			c <<= 6;			c += (u8int)s[i++];			csz++;		}while(i < sz && !isutf(s[i]));		c -= offsetsFromUTF8[csz-1]; 		if(c == ch)			return s+lasti;		lasti = i;		(*charn)++;	}	return nil;} /* based on the valid_utf8 routine from the PCRE library by Philip Hazel    length is in bytes, since without knowing whether the string is valid   it's hard to know how many characters there are! */boolu8_isvalid(const char *str, int length){	const u8int *p, *pend = (const u8int*)str + length;	u8int c;	int ab; 	for(p = (const u8int*)str; p < pend; p++){		c = *p;		if(c < 128)			continue;		if((c & 0xc0) != 0xc0)			return false;		ab = trailingBytesForUTF8[c];		if(length < ab)			return false;		length -= ab; 		p++;		/* Check top bits in the second byte */		if((*p & 0xc0) != 0x80)			return false; 		/* Check for overlong sequences for each different length */		switch(ab){			/* Check for xx00 000x */		case 1:			if((c & 0x3e) == 0)				return false;			continue; /* We know there aren't any more bytes to check */ 			/* Check for 1110 0000, xx0x xxxx */		case 2:			if(c == 0xe0 && (*p & 0x20) == 0)				return false;			break; 			/* Check for 1111 0000, xx00 xxxx */		case 3:			if(c == 0xf0 && (*p & 0x30) == 0)				return false;			break; 			/* Check for 1111 1000, xx00 0xxx */		case 4:			if(c == 0xf8 && (*p & 0x38) == 0)				return false;			break; 			/* Check for leading 0xfe or 0xff and then for 1111 1100, xx00 00xx */		case 5:			if(c == 0xfe || c == 0xff || (c == 0xfc && (*p & 0x3c) == 0))				return false;			break;		} 		/* Check for valid bytes after the 2nd, if any; all must start 10 */		while(--ab > 0)			if((*(++p) & 0xc0) != 0x80)				return false;	} 	return true;} voidu8_reverse(char *dest, char *src, usize len){	usize si, di; 	if(len == 0)		return; 	dest[di = len] = '\0';	for(si = 0; si < len;){		switch((u8int)src[si]>>4){		case 0xf:			di -= 4;			dest[di+0] = src[si++];			dest[di+1] = src[si++];			dest[di+2] = src[si++];			dest[di+3] = src[si++];			break;		case 0xe:			di -= 3;			dest[di+0] = src[si++];			dest[di+1] = src[si++];			dest[di+2] = src[si++];			break;		case 0xd:		case 0xc:			di -= 2;			dest[di+0] = src[si++];			dest[di+1] = src[si++];			break;		default:			di--;			dest[di+0] = src[si++];			break;		}	}}