The JSON output format has been changed to support valid utf8 and non-utf8 binary data. Valid utf8 has a '+' prefix and store the data as a standard JSON string. Binary has a '$' prefix and store the data in base64. fixes #1
Unified · Split
| @@ -2,6 +2,7 @@ package wal · 1 unmodified line | ||
| 2 | 2 | |
| 3 | 3 | import ( |
| 4 | 4 | "bufio" |
| 5 | + "encoding/base64" | |
| 5 | 6 | "encoding/binary" |
| 6 | 7 | "encoding/json" |
| 7 | 8 | "errors" |
| @@ -12,6 +13,8 @@ import ( · 4 unmodified lines | ||
| 12 | 13 | "path/filepath" |
| 13 | 14 | "strconv" |
| 14 | 15 | "strings" |
| 16 | + "unicode/utf8" | |
| 17 | + "unsafe" | |
| 15 | 18 | ) |
| 16 | 19 | |
| 17 | 20 | var ( |
| @@ -343,22 +346,20 @@ func appendJSONEntry(dst []byte, index uint64, data []byte) []byte { · 325 unmodified lines | ||
| 343 | 346 | dst = append(dst, `{"index":"`...) |
| 344 | 347 | dst = strconv.AppendUint(dst, index, 10) |
| 345 | 348 | dst = append(dst, `","data":`...) |
| 346 | − dst = appendJSONString(dst, data) | |
| 349 | + dst = appendJSONData(dst, data) | |
| 347 | 350 | dst = append(dst, '}', '\n') |
| 348 | 351 | return dst |
| 349 | 352 | } |
| 350 | 353 | |
| 351 | −func appendJSONString(dst []byte, s []byte) []byte { | |
| 352 | − for i := 0; i < len(s); i++ { | |
| 353 | − if s[i] < ' ' || s[i] == '\\' || s[i] == '"' || s[i] > 126 { | |
| 354 | − d, _ := json.Marshal(string(s)) | |
| 355 | − return append(dst, d...) | |
| 356 | − } | |
| 354 | +func appendJSONData(dst []byte, s []byte) []byte { | |
| 355 | + if utf8.Valid(s) { | |
| 356 | + b, _ := json.Marshal(*(*string)(unsafe.Pointer(&s))) | |
| 357 | + dst = append(dst, '"', '+') | |
| 358 | + return append(dst, b[1:]...) | |
| 357 | 359 | } |
| 358 | − dst = append(dst, '"') | |
| 359 | − dst = append(dst, s...) | |
| 360 | − dst = append(dst, '"') | |
| 361 | − return dst | |
| 360 | + dst = append(dst, '"', '$') | |
| 361 | + dst = append(dst, base64.URLEncoding.EncodeToString(s)...) | |
| 362 | + return append(dst, '"') | |
| 362 | 363 | } |
| 363 | 364 | |
| 364 | 365 | func appendBinaryEntry(dst []byte, index uint64, data []byte) []byte { |
| @@ -615,11 +616,21 @@ func readEntry(rd *bufio.Reader, frmt LogFormat, discardData bool) ( · 250 unmodified lines | ||
| 615 | 616 | return 0, nil, ErrCorrupt |
| 616 | 617 | } |
| 617 | 618 | s, ok := m["data"] |
| 618 | − if !ok { | |
| 619 | + if !ok || len(s) == 0 { | |
| 619 | 620 | return 0, nil, ErrCorrupt |
| 620 | 621 | } |
| 621 | 622 | if !discardData { |
| 622 | − data = []byte(s) | |
| 623 | + switch s[0] { | |
| 624 | + case '$': | |
| 625 | + data, err = base64.URLEncoding.DecodeString(s[1:]) | |
| 626 | + if err != nil { | |
| 627 | + return 0, nil, ErrCorrupt | |
| 628 | + } | |
| 629 | + case '+': | |
| 630 | + data = []byte(s[1:]) | |
| 631 | + default: | |
| 632 | + return 0, nil, ErrCorrupt | |
| 633 | + } | |
| 623 | 634 | } |
| 624 | 635 | return index, data, nil |
| 625 | 636 | } |
| @@ -613,3 +613,28 @@ func makeOpts(segSize int, dur Durability, lf LogFormat) *Options { · 612 unmodified lines | ||
| 613 | 613 | opts.LogFormat = lf |
| 614 | 614 | return &opts |
| 615 | 615 | } |
| 616 | + | |
| 617 | +// https://github.com/tidwall/wal/issues/1 | |
| 618 | +func TestIssue1(t *testing.T) { | |
| 619 | + in := []byte{0, 0, 0, 0, 0, 0, 0, 1, 37, 108, 131, 178, 151, 17, 77, 32, | |
| 620 | + 27, 48, 23, 159, 63, 14, 240, 202, 206, 151, 131, 98, 45, 165, 151, 67, | |
| 621 | + 38, 180, 54, 23, 138, 238, 246, 16, 0, 0, 0, 0} | |
| 622 | + opts := *DefaultOptions | |
| 623 | + opts.LogFormat = JSON | |
| 624 | + os.RemoveAll("testlog") | |
| 625 | + l, err := Open("testlog", &opts) | |
| 626 | + if err != nil { | |
| 627 | + t.Fatal(err) | |
| 628 | + } | |
| 629 | + defer l.Close() | |
| 630 | + if err := l.Write(1, in); err != nil { | |
| 631 | + t.Fatal(err) | |
| 632 | + } | |
| 633 | + out, err := l.Read(1) | |
| 634 | + if err != nil { | |
| 635 | + t.Fatal(err) | |
| 636 | + } | |
| 637 | + if string(in) != string(out) { | |
| 638 | + t.Fatal("data mismatch") | |
| 639 | + } | |
| 640 | +} | |