The JSON output format has been changed to support valid utf8 and non-utf8 binary data. Valid utf8 has a '+' prefix and store the data as a standard JSON string. Binary has a '$' prefix and store the data in base64. fixes #1
Unified · Split
| @@ -2,6 +2,7 @@ package wal · 1 unmodified line | |||
| 2 | 2 | ||
| 3 | import ( | 3 | import ( |
| 4 | "bufio" | 4 | "bufio" |
| 5 | "encoding/base64" | ||
| 5 | "encoding/binary" | 6 | "encoding/binary" |
| 6 | "encoding/json" | 7 | "encoding/json" |
| 7 | "errors" | 8 | "errors" |
| @@ -12,6 +13,8 @@ import ( · 4 unmodified lines | |||
| 12 | "path/filepath" | 13 | "path/filepath" |
| 13 | "strconv" | 14 | "strconv" |
| 14 | "strings" | 15 | "strings" |
| 16 | "unicode/utf8" | ||
| 17 | "unsafe" | ||
| 15 | ) | 18 | ) |
| 16 | 19 | ||
| 17 | var ( | 20 | var ( |
| @@ -343,22 +346,20 @@ func appendJSONEntry(dst []byte, index uint64, data []byte) []byte { · 325 unmodified lines | |||
| 343 | dst = append(dst, `{"index":"`...) | 346 | dst = append(dst, `{"index":"`...) |
| 344 | dst = strconv.AppendUint(dst, index, 10) | 347 | dst = strconv.AppendUint(dst, index, 10) |
| 345 | dst = append(dst, `","data":`...) | 348 | dst = append(dst, `","data":`...) |
| 346 | dst = appendJSONString(dst, data) | 349 | dst = appendJSONData(dst, data) |
| 347 | dst = append(dst, '}', '\n') | 350 | dst = append(dst, '}', '\n') |
| 348 | return dst | 351 | return dst |
| 349 | } | 352 | } |
| 350 | 353 | ||
| 351 | func appendJSONString(dst []byte, s []byte) []byte { | 354 | func appendJSONData(dst []byte, s []byte) []byte { |
| 352 | for i := 0; i < len(s); i++ { | 355 | if utf8.Valid(s) { |
| 353 | if s[i] < ' ' || s[i] == '\\' || s[i] == '"' || s[i] > 126 { | 356 | b, _ := json.Marshal(*(*string)(unsafe.Pointer(&s))) |
| 354 | d, _ := json.Marshal(string(s)) | 357 | dst = append(dst, '"', '+') |
| 355 | return append(dst, d...) | 358 | return append(dst, b[1:]...) |
| 356 | } | ||
| 357 | } | 359 | } |
| 358 | dst = append(dst, '"') | 360 | dst = append(dst, '"', '$') |
| 359 | dst = append(dst, s...) | 361 | dst = append(dst, base64.URLEncoding.EncodeToString(s)...) |
| 360 | dst = append(dst, '"') | 362 | return append(dst, '"') |
| 361 | return dst | ||
| 362 | } | 363 | } |
| 363 | 364 | ||
| 364 | func appendBinaryEntry(dst []byte, index uint64, data []byte) []byte { | 365 | func appendBinaryEntry(dst []byte, index uint64, data []byte) []byte { |
| @@ -615,11 +616,21 @@ func readEntry(rd *bufio.Reader, frmt LogFormat, discardData bool) ( · 250 unmodified lines | |||
| 615 | return 0, nil, ErrCorrupt | 616 | return 0, nil, ErrCorrupt |
| 616 | } | 617 | } |
| 617 | s, ok := m["data"] | 618 | s, ok := m["data"] |
| 618 | if !ok { | 619 | if !ok || len(s) == 0 { |
| 619 | return 0, nil, ErrCorrupt | 620 | return 0, nil, ErrCorrupt |
| 620 | } | 621 | } |
| 621 | if !discardData { | 622 | if !discardData { |
| 622 | data = []byte(s) | 623 | switch s[0] { |
| 624 | case '$': | ||
| 625 | data, err = base64.URLEncoding.DecodeString(s[1:]) | ||
| 626 | if err != nil { | ||
| 627 | return 0, nil, ErrCorrupt | ||
| 628 | } | ||
| 629 | case '+': | ||
| 630 | data = []byte(s[1:]) | ||
| 631 | default: | ||
| 632 | return 0, nil, ErrCorrupt | ||
| 633 | } | ||
| 623 | } | 634 | } |
| 624 | return index, data, nil | 635 | return index, data, nil |
| 625 | } | 636 | } |
| @@ -613,3 +613,28 @@ func makeOpts(segSize int, dur Durability, lf LogFormat) *Options { · 612 unmodified lines | |||
| 613 | opts.LogFormat = lf | 613 | opts.LogFormat = lf |
| 614 | return &opts | 614 | return &opts |
| 615 | } | 615 | } |
| 616 | |||
| 617 | // https://github.com/tidwall/wal/issues/1 | ||
| 618 | func TestIssue1(t *testing.T) { | ||
| 619 | in := []byte{0, 0, 0, 0, 0, 0, 0, 1, 37, 108, 131, 178, 151, 17, 77, 32, | ||
| 620 | 27, 48, 23, 159, 63, 14, 240, 202, 206, 151, 131, 98, 45, 165, 151, 67, | ||
| 621 | 38, 180, 54, 23, 138, 238, 246, 16, 0, 0, 0, 0} | ||
| 622 | opts := *DefaultOptions | ||
| 623 | opts.LogFormat = JSON | ||
| 624 | os.RemoveAll("testlog") | ||
| 625 | l, err := Open("testlog", &opts) | ||
| 626 | if err != nil { | ||
| 627 | t.Fatal(err) | ||
| 628 | } | ||
| 629 | defer l.Close() | ||
| 630 | if err := l.Write(1, in); err != nil { | ||
| 631 | t.Fatal(err) | ||
| 632 | } | ||
| 633 | out, err := l.Read(1) | ||
| 634 | if err != nil { | ||
| 635 | t.Fatal(err) | ||
| 636 | } | ||
| 637 | if string(in) != string(out) { | ||
| 638 | t.Fatal("data mismatch") | ||
| 639 | } | ||
| 640 | } | ||