#include "inc/priv.h" #include "inc/operators.h" #include "inc/cvalues.h" #include "inc/print.h" #include "inc/read.h" #include "inc/equal.h" #include "inc/io.h" #include "inc/str.h" static sl_v basesym; static const u8int ws[] = { 0x20, 0x0a, 0x09, 0x0b, 0x0c, 0x0d, 0xc2, 0x85, 0xc2, 0xa0, 0xe1, 0x9a, 0x80, 0xe2, 0x80, 0x80, 0xe2, 0x80, 0x81, 0xe2, 0x80, 0x82, 0xe2, 0x80, 0x83, 0xe2, 0x80, 0x84, 0xe2, 0x80, 0x85, 0xe2, 0x80, 0x86, 0xe2, 0x80, 0x87, 0xe2, 0x80, 0x88, 0xe2, 0x80, 0x89, 0xe2, 0x80, 0x8a, 0xe2, 0x80, 0xa8, 0xe2, 0x80, 0xa9, 0xe2, 0x80, 0xaf, 0xe2, 0x81, 0x9f, 0xe3, 0x80, 0x80, 0x00 }; static const u32int offsetsFromUTF8[6] = { 0x00000000U, 0x00003080U, 0x000E2080U, 0x03C82080U, 0xFA082080U, 0x82082080U }; static const u8int trailingBytesForUTF8[256] = { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 }; /* returns length of next utf-8 sequence */ usize u8_seqlen(const char *s) { return trailingBytesForUTF8[(u8int)s[0]] + 1; } /* modified runenlen from 9front */ usize u8_runelen(const char *s, usize m, bool *err) { const char *es = s + m; usize n; *err = false; for(n = 0; s < es; n++) { if(*(const u8int*)s < Runeself){ s++; continue; } if(sl_unlikely(!fullrune(s, es-s))){ *err = true; break; } Rune r; s += chartorune(&r, s); if(sl_unlikely(r == Runeerror)){ *err = true; break; } } return n; } usize u8_strnlen(const char *s, usize n, ssize maxwidth, ssize *outwidth, bool *err) { usize i, len = 0; ssize totw = 0, clw = 0; Rune lastr = Runeerror; bool skip = false, incluster = false; *err = false; for(i = 0; i < n;){ if(sl_unlikely(!fullrune(s+i, n-i))){ *err = true; break; } Rune r; usize dt = chartorune(&r, s+i); if(sl_unlikely(r == Runeerror)){ *err = true; break; } bool boundary = !skip && !(r == Runevs16 && lastr != Runeerror); if(boundary && incluster){ if(maxwidth >= 0 && totw + clw > maxwidth) break; totw += clw; len = i; clw = 0; } if(boundary) incluster = true; int w; if(skip) skip = false; else if(r == Runezwj) skip = true; else if(r == Runevs16 && lastr != Runeerror){ clw += sl_wcwidth_vs16(lastr); lastr = Runeerror; }else if(sl_likely((w = sl_wcwidth(r)) > 0)){ lastr = r; clw += w; }else if(w < 0){ *err = true; break; } i += dt; } if(incluster && (maxwidth < 0 || totw + clw <= maxwidth)){ totw += clw; len = i; } *outwidth = totw; return len; } ssize u8_strwidth(const char *s, usize n, bool *err) { ssize w; u8_strnlen(s, n, -1, &w, err); return w; } /* next character without NUL character terminator */ Rune u8_nextmemchar(const char *s, usize *i) { usize sz = trailingBytesForUTF8[(u8int)s[*i]]; Rune ch = 0; for(usize j = 0; j <= sz; j++){ ch <<= 6; ch += (u8int)s[(*i)++]; }; return ch - offsetsFromUTF8[sz]; } bool octal_digit(char c) { return c >= '0' && c <= '7'; } bool hex_digit(char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); } char read_escape_control_char(char c) { switch(c){ case 'n': return '\n'; case 't': return '\t'; case 'a': return '\a'; case 'b': return '\b'; case 'e': return 0x1b; case 'f': return '\f'; case 'r': return '\r'; case 'v': return '\v'; } return c; } static inline int buf_put2c(char *buf, const char *src) { buf[0] = src[0]; buf[1] = src[1]; buf[2] = '\0'; return 2; } int u8_escape_rune(char *buf, usize sz, Rune ch) { assert(sz > 12); if(ch == '\\') return buf_put2c(buf, "\\\\"); if(ch >= 0x20 && ch < 0x7f){ buf[0] = ch; buf[1] = '\0'; return 1; } if(ch > 0xffff) return snprintf(buf, sz, "\\U%.8"PRIx32, ch); if(ch >= Runeself) return snprintf(buf, sz, "\\u%04"PRIx32, ch); switch(ch){ case '\n': return buf_put2c(buf, "\\n"); case '\t': return buf_put2c(buf, "\\t"); case '\a': return buf_put2c(buf, "\\a"); case '\b': return buf_put2c(buf, "\\b"); case 0x1b: return buf_put2c(buf, "\\e"); case '\f': return buf_put2c(buf, "\\f"); case '\r': return buf_put2c(buf, "\\r"); case '\v': return buf_put2c(buf, "\\v"); } return snprintf(buf, sz, "\\x%02"PRIx32, ch); } usize u8_escape(char *buf, usize sz, const char *src, usize *pi, usize end, bool escape_quotes, bool ascii) { usize i = *pi, i0; Rune ch; char *start = buf; char *blim = start + sz-12; assert(sz > 12); while(i < end && buf < blim){ // sz-12: leaves room for longest escape sequence if(escape_quotes && src[i] == '"'){ buf += buf_put2c(buf, "\\\""); i++; }else if(src[i] == '\\'){ buf += buf_put2c(buf, "\\\\"); i++; }else{ i0 = i; ch = u8_nextmemchar(src, &i); if(ascii || !sl_iswprint(ch)){ buf += u8_escape_rune(buf, sz - (buf-start), ch); }else{ i = i0; do{ *buf++ = src[i++]; }while(i < end && !isutf(src[i])); } } } *buf++ = '\0'; *pi = i; return buf - start; } u8int* u8_memchr(u8int *s, Rune ch, usize sz, usize *charn) { usize i = 0, lasti = 0; Rune c; int csz; *charn = 0; while(i < sz){ c = csz = 0; do{ c <<= 6; c += (u8int)s[i++]; csz++; }while(i < sz && !isutf(s[i])); c -= offsetsFromUTF8[csz-1]; if(c == ch) return s+lasti; lasti = i; (*charn)++; } return nil; } /* based on the valid_utf8 routine from the PCRE library by Philip Hazel length is in bytes, since without knowing whether the string is valid it's hard to know how many characters there are! */ bool u8_isvalid(const char *str, int length) { const u8int *p, *pend = (const u8int*)str + length; u8int c; int ab; for(p = (const u8int*)str; p < pend; p++){ c = *p; if(sl_likely(c < Runeself)) continue; if(sl_unlikely((c & 0xc0) != 0xc0)) return false; ab = trailingBytesForUTF8[c]; if(sl_unlikely(length < ab)) return false; length -= ab; p++; /* Check top bits in the second byte */ if(sl_unlikely((*p & 0xc0) != 0x80)) return false; /* Check for overlong sequences for each different length */ switch(ab){ /* Check for xx00 000x */ case 1: if(sl_unlikely((c & 0x3e) == 0)) return false; continue; /* We know there aren't any more bytes to check */ /* Check for 1110 0000, xx0x xxxx */ case 2: if(sl_unlikely(c == 0xe0 && (*p & 0x20) == 0)) return false; break; /* Check for 1111 0000, xx00 xxxx */ case 3: if(sl_unlikely(c == 0xf0 && (*p & 0x30) == 0)) return false; break; /* Check for 1111 1000, xx00 0xxx */ case 4: if(sl_unlikely(c == 0xf8 && (*p & 0x38) == 0)) return false; break; /* Check for leading 0xfe or 0xff and then for 1111 1100, xx00 00xx */ case 5: if(sl_unlikely(c == 0xfe || c == 0xff || (c == 0xfc && (*p & 0x3c) == 0))) return false; break; default: abort(); } /* Check for valid bytes after the 2nd, if any; all must start 10 */ while(--ab > 0) if(sl_unlikely((*(++p) & 0xc0) != 0x80)) return false; } return true; } void u8_reverse(char *dest, const char *src, usize len) { usize si, di; if(len == 0) return; dest[di = len] = '\0'; for(si = 0; si < len;){ switch((u8int)src[si]>>4){ case 0xf: di -= 4; dest[di+0] = src[si++]; dest[di+1] = src[si++]; dest[di+2] = src[si++]; dest[di+3] = src[si++]; break; case 0xe: di -= 3; dest[di+0] = src[si++]; dest[di+1] = src[si++]; dest[di+2] = src[si++]; break; case 0xd: case 0xc: di -= 2; dest[di+0] = src[si++]; dest[di+1] = src[si++]; break; default: di--; dest[di+0] = src[si++]; break; } } } int isdigit_base(char c, int base) { if(sl_unlikely(base < 11)) return c >= '0' && c < '0'+base; if(sl_unlikely(base == 32)) return (c >= 'A' && c <= 'Z') || (c >= '2' && c <= '7'); return (c >= '0' && c <= '9') || (c >= 'a' && c < 'a'+base-10) || (c >= 'A' && c < 'A'+base-10); } BUILTIN("str?", strp) { argcount(1); return isstr(args[0]) ? sl_t : sl_nil; } BUILTIN("str-width", str_width) { argcount(1); if(sl_unlikely(sl_isrune(args[0]))){ int w = sl_wcwidth(torune(args[0])); return w < 0 ? sl_nil : fixnum(w); } if(sl_unlikely(!isstr(args[0]))) bthrow(type_error(nil, "str", args[0])); char *str = tostr(args[0]); usize len = cvalue_len(args[0]); bool err = false; ssize w = u8_strwidth(str, len, &err); return err ? sl_nil : fixnum(w); } BUILTIN("str-reverse", str_reverse) { argcount(1); if(sl_unlikely(sl_isrune(args[0]))) return args[0]; if(sl_unlikely(!isstr(args[0]))) bthrow(type_error(nil, "str", args[0])); usize len = cvalue_len(args[0]); sl_v ns = cvalue_str(len); u8_reverse(cvalue_data(ns), cvalue_data(args[0]), len); return ns; } BUILTIN("str", str) { if(sl_unlikely(nargs == 1) && isstr(args[0])) return args[0]; ios s; if(sl_unlikely(ios_mem(&s, 0) == nil)) bthrow(error(sl_errmem, "could not allocate in-memory io")); sl_v oldpr = sym_value(sl_printreadablysym); // FIXME: gc if cvalue sl_v oldpp = sym_value(sl_printprettysym); // FIXME: gc if cvalue set(sl_printreadablysym, sl_nil); set(sl_printprettysym, sl_nil); sl_v arg; int i; sl.pr.f = &s; FOR_ARGS(i, 0, arg, args){ USED(arg); sl_print(args[i]); } set(sl_printreadablysym, oldpr); set(sl_printprettysym, oldpp); sl_cv cv = { .type = sl_iotype, {.data = &s}, .len = s.size, }; sl_v buf = tagptr(&cv, TAG_CVALUE); sl_v outp = io_to_str(&buf); ios_free(&s); return outp; } BUILTIN("str-split", str_split) { argcount_least(1); char *s = tostr(args[0]); usize len = cvalue_len(args[0]); // split on whitespace by default const char *delim0 = (const char*)ws, *delim = delim0; usize dlen = 5; int n = 1; // second is either a :trim or a separator if(nargs > n && args[n] != sl_trimsym){ delim = tostr(args[n]); dlen = cvalue_len(args[n]); n++; } bool trim = false; // it can only be a :trim X now if(sl_unlikely(nargs > n)){ if(sl_unlikely(args[n] != sl_trimsym)) bthrow(error(sl_errarg, "invalid argument at position %d", n)); n++; if(nargs <= n) argcount(n+1); trim = args[n] != sl_nil; } usize ssz, tokend, tokstart, i = 0; sl_v first = sl_nil, c = sl_nil, last; usize junk; sl_gc_handle(&first); sl_gc_handle(&last); do{ // find and allocate next token tokstart = tokend = i; while(i < len && !u8_memchr((u8int*)delim, u8_nextmemchar(s, &i), dlen, &junk)) tokend = i; ssz = tokend - tokstart; if(ssz == 0 && trim) continue; last = c; // save previous cons cell c = mk_cons(cvalue_str(ssz), sl_nil); if(delim != delim0){ // we've done allocation; reload movable pointers s = cvalue_data(args[0]); delim = cvalue_data(args[1]); } if(ssz) memmove(cvalue_data(car_(c)), &s[tokstart], ssz); // link new cell if(last == sl_nil) first = c; // first time, save first cons else ptr(sl_cons, last)->cdr = c; // note this tricky condition: if the string ends with a // delimiter, we need to go around one more time to add an // empty string. this happens when (i == len && tokend < i) }while(i < len || (i == len && tokend != i)); sl_free_gc_handles(2); return first; } usize rune_index(sl_v str, usize runeidx) { char *s = tostr(str); sl_cv *pcv = ptr(sl_cv, str); usize startbytes, n; if(!isinlined(pcv) && pcv->startrune <= runeidx){ startbytes = pcv->startbytes; n = pcv->startrune; }else{ startbytes = 0; n = 0; } for(; n < runeidx && startbytes < pcv->len; n++) startbytes += u8_seqlen(s+startbytes); if(sl_unlikely(n != runeidx)) bthrow(bounds_error(str, fixnum(runeidx))); if(!isinlined(pcv)){ pcv->startbytes = startbytes; pcv->startrune = runeidx; } return startbytes; } BUILTIN("str-sub", str_sub) { argcount_least(2); argcount_most(3); char *s = tostr(args[0]); usize lenbytes = cvalue_len(args[0]), n = tosize(args[1]); usize startbytes = rune_index(args[0], n), endbytes = lenbytes; if(nargs == 3 && args[2] != sl_nil){ usize endrune = tosize(args[2]); for(endbytes = startbytes; n < endrune && endbytes < lenbytes; n++) endbytes += u8_seqlen(s+endbytes); if(sl_unlikely(n != endrune)) bthrow(bounds_error(args[0], args[2])); } sl_v ns = cvalue_str(endbytes-startbytes); s = cvalue_data(args[0]); // reload after alloc memmove(cvalue_data(ns), s+startbytes, endbytes-startbytes); return ns; } BUILTIN("str-rune", str_rune) { argcount(2); usize startbytes = rune_index(args[0], tosize(args[1])); if(sl_unlikely(startbytes >= cvalue_len(args[0]))) bthrow(bounds_error(args[0], args[1])); char *s = tostr(args[0]); Rune r; chartorune(&r, s+startbytes); return mk_rune(r); } BUILTIN("rune->codepoint", rune_2_codepoint) { argcount(1); if(sl_unlikely(!sl_isrune(args[0]))) bthrow(type_error(nil, "rune", args[0])); return fixnum(torune(args[0])); } BUILTIN("rune-upcase", rune_upcase) { argcount(1); if(sl_unlikely(!sl_isrune(args[0]))) bthrow(type_error(nil, "rune", args[0])); return mk_rune(toupperrune(torune(args[0]))); } BUILTIN("rune-downcase", rune_downcase) { argcount(1); if(sl_unlikely(!sl_isrune(args[0]))) bthrow(type_error(nil, "rune", args[0])); return mk_rune(tolowerrune(torune(args[0]))); } BUILTIN("rune-titlecase", rune_titlecase) { argcount(1); if(sl_unlikely(!sl_isrune(args[0]))) bthrow(type_error(nil, "rune", args[0])); return mk_rune(totitlerune(torune(args[0]))); } BUILTIN("rune-alphabetic?", rune_alphabeticp) { argcount(1); if(sl_unlikely(!sl_isrune(args[0]))) bthrow(type_error(nil, "rune", args[0])); return isalpharune(torune(args[0])) ? sl_t : sl_nil; } BUILTIN("rune-lower-case?", rune_lower_casep) { argcount(1); if(sl_unlikely(!sl_isrune(args[0]))) bthrow(type_error(nil, "rune", args[0])); return islowerrune(torune(args[0])) ? sl_t : sl_nil; } BUILTIN("rune-upper-case?", rune_upper_casep) { argcount(1); if(sl_unlikely(!sl_isrune(args[0]))) bthrow(type_error(nil, "rune", args[0])); return isupperrune(torune(args[0])) ? sl_t : sl_nil; } BUILTIN("rune-title-case?", rune_title_casep) { argcount(1); if(sl_unlikely(!sl_isrune(args[0]))) bthrow(type_error(nil, "rune", args[0])); return istitlerune(torune(args[0])) ? sl_t : sl_nil; } BUILTIN("rune-numeric?", rune_numericp) { argcount(1); if(sl_unlikely(!sl_isrune(args[0]))) bthrow(type_error(nil, "rune", args[0])); return isdigitrune(torune(args[0])) ? sl_t : sl_nil; } BUILTIN("rune-whitespace?", rune_whitespacep) { argcount(1); if(sl_unlikely(!sl_isrune(args[0]))) bthrow(type_error(nil, "rune", args[0])); return isspacerune(torune(args[0])) ? sl_t : sl_nil; } BUILTIN("str-find", str_find) { argcount_least(2); argcount_most(3); usize startrune = nargs == 3 ? tosize(args[2]) : 0; char *s = tostr(args[0]); sl_cv *pcv = ptr(sl_cv, args[0]); usize sbytes = pcv->len; if(sbytes == 0) return sl_nil; // needle char *nd; usize ndbytes; sl_v v = args[1]; char rbuf[UTFmax+1]; if(sl_isrune(v)){ Rune r = torune(v); ndbytes = runetochar(rbuf, &r); nd = rbuf; nd[ndbytes] = 0; }else if(isstr(v)){ sl_cv *cv = ptr(sl_cv, v); ndbytes = cv_len(cv); nd = (char*)cv_data(cv); }else{ bthrow(type_error("needle", "(or str rune)", args[1])); } if(ndbytes == 0) return size_wrap(startrune); if(ndbytes > sbytes) return sl_nil; usize i = rune_index(args[0], startrune), n; // now search for the needle for(n = startrune; i < sbytes-ndbytes+1; n++){ if(s[i] == nd[0] && memcmp(&s[i+1], nd+1, ndbytes-1) == 0) return size_wrap(n); i += u8_seqlen(s+i); } return sl_nil; } static int get_radix_arg(sl_v arg) { usize radix = tosize(arg); if(sl_unlikely(radix < 2 || radix > 36)) cthrow(error(sl_errarg, "invalid radix: %"PRIuPTR, (uintptr)radix), arg); return radix; } BUILTIN("str->num", str_2_num) { sl_v keys[] = {basesym}, values[nelem(keys)]; nargs = sl_process_args(args, 1, nargs, keys, nelem(keys), values); argcount(1); char *s = tostr(args[0]), *e; for(;;){ Rune r; int n = chartorune(&r, s); if(r == Runeerror) return sl_nil; if(!isspacerune(r)) break; s += n; } sl_v n; int base = 10; if(nonnil(values[0])) base = get_radix_arg(values[0]); else if(s[0] == '#'){ int c = tolower((u8int)s[1]); if(((c == 'b' && (base = 2)) || (c == 'o' && (base = 8)) || (c == 'd' && (base = 10)) || (c == 'x' && (base = 16))) && (isdigit_base(s[2], base) || s[2] == '-')){ s += 2; }else if(isdigit((u8int)s[1])){ int r = strtol(s+1, &e, 10); if(*e == 'r' || *e == 'R'){ if(sl_unlikely(r < 2 || r > 36)) bthrow(error(sl_errarg, "invalid radix: %d", r)); base = r; s = e+1; } }else if((u8int)s[1] < Runeself) bthrow(error(sl_errarg, "invalid radix prefix: %c", s[1])); else bthrow(error(sl_errarg, "invalid radix prefix")); } if(sl_unlikely(!sl_read_numtok(s, &n, base))) return sl_nil; return n; } BUILTIN("str-utf8?", str_utf8p) { argcount(1); char *s = tostr(args[0]); usize len = cvalue_len(args[0]); return u8_isvalid(s, len) ? sl_t : sl_nil; } void str_init(void) { basesym = mk_csym(":base"); }