History

select: Add more compression formats (#13142)

Support Zstandard, LZ4, S2, and snappy as additional compression formats for S3 Select.

Klaus Post committed 5 years ago

Commit b2c92cd1 parent 42b1d926 files · +62 −30Browse files

Unified · Split

6 files
  1. docs/select/README.md +2 −1
  2. go.mod +1 −1
  3. go.sum +2 −2
  4. internal/s3select/errors.go +5 −12
  5. internal/s3select/progress.go +24 −7
  6. internal/s3select/select.go +28 −7
docs/select/README.md+2−1
@@ -5,7 +5,8 @@ You can use the Select API to query objects with following features: · 4 unmodified lines
5 5
6- Objects must be in CSV, JSON, or Parquet(*) format. 6- Objects must be in CSV, JSON, or Parquet(*) format.
7- UTF-8 is the only encoding type the Select API supports.7- UTF-8 is the only encoding type the Select API supports.
8- GZIP or BZIP2 - CSV and JSON files can be compressed using GZIP or BZIP2. The Select API supports columnar compression for Parquet using GZIP, Snappy, LZ4. Whole object compression is not supported for Parquet objects.8- GZIP or BZIP2 - CSV and JSON files can be compressed using GZIP, BZIP2, [ZSTD](https://facebook.github.io/zstd/), and streaming formats of [LZ4](https://lz4.github.io/lz4/), [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) and [SNAPPY](http://google.github.io/snappy/).
9- Parquet API supports columnar compression for using GZIP, Snappy, LZ4. Whole object compression is not supported for Parquet objects.
9- Server-side encryption - The Select API supports querying objects that are protected with server-side encryption.10- Server-side encryption - The Select API supports querying objects that are protected with server-side encryption.
10 11
11Type inference and automatic conversion of values is performed based on the context when the value is un-typed (such as when reading CSV data). If present, the CAST function overrides automatic conversion.12Type inference and automatic conversion of values is performed based on the context when the value is un-typed (such as when reading CSV data). If present, the CAST function overrides automatic conversion.
go.mod+1−1
@@ -32,7 +32,7 @@ require ( · 31 unmodified lines
32 github.com/gorilla/mux v1.8.032 github.com/gorilla/mux v1.8.0
33 github.com/jcmturner/gokrb5/v8 v8.4.233 github.com/jcmturner/gokrb5/v8 v8.4.2
34 github.com/json-iterator/go v1.1.1134 github.com/json-iterator/go v1.1.11
35 github.com/klauspost/compress v1.13.435 github.com/klauspost/compress v1.13.5
36 github.com/klauspost/cpuid/v2 v2.0.436 github.com/klauspost/cpuid/v2 v2.0.4
37 github.com/klauspost/pgzip v1.2.537 github.com/klauspost/pgzip v1.2.5
38 github.com/klauspost/readahead v1.3.138 github.com/klauspost/readahead v1.3.1
go.sum+2−2
@@ -883,8 +883,8 @@ github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs · 882 unmodified lines
883github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=883github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
884github.com/klauspost/compress v1.11.12/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=884github.com/klauspost/compress v1.11.12/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
885github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=885github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
886github.com/klauspost/compress v1.13.4 h1:0zhec2I8zGnjWcKyLl6i3gPqKANCCn5e9xmviEEeX6s=886github.com/klauspost/compress v1.13.5 h1:9O69jUPDcsT9fEm74W92rZL9FQY7rCdaXVneq+yyzl4=
887github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=887github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
888github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=888github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
889github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=889github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
890github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=890github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
internal/s3select/errors.go+5−12
@@ -17,6 +17,8 @@ · 16 unmodified lines
17 17
18package s3select18package s3select
19 19
20import "strings"
21
20// SelectError - represents s3 select error specified in22// SelectError - represents s3 select error specified in
21// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html#RESTObjectSELECTContent-responses-special-errors.23// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html#RESTObjectSELECTContent-responses-special-errors.
22type SelectError interface {24type SelectError interface {
@@ -66,25 +68,16 @@ func errMalformedXML(err error) *s3Error { · 43 unmodified lines
66func errInvalidCompressionFormat(err error) *s3Error {68func errInvalidCompressionFormat(err error) *s3Error {
67 return &s3Error{69 return &s3Error{
68 code: "InvalidCompressionFormat",70 code: "InvalidCompressionFormat",
69 message: "The file is not in a supported compression format. Only GZIP and BZIP2 are supported.",71 message: "The file is not in a supported compression format. GZIP, BZIP2, ZSTD, LZ4, S2 and SNAPPY are supported.",
70 statusCode: 400,
71 cause: err,
72 }
73}
74
75func errInvalidBZIP2CompressionFormat(err error) *s3Error {
76 return &s3Error{
77 code: "InvalidCompressionFormat",
78 message: "BZIP2 is not applicable to the queried object. Please correct the request and try again.",
79 statusCode: 400,72 statusCode: 400,
80 cause: err,73 cause: err,
81 }74 }
82}75}
83 76
84func errInvalidGZIPCompressionFormat(err error) *s3Error {77func errInvalidCompression(err error, t CompressionType) *s3Error {
85 return &s3Error{78 return &s3Error{
86 code: "InvalidCompressionFormat",79 code: "InvalidCompressionFormat",
87 message: "GZIP is not applicable to the queried object. Please correct the request and try again.",80 message: strings.ToUpper(string(t)) + " is not applicable to the queried object. Please correct the request and try again.",
88 statusCode: 400,81 statusCode: 400,
89 cause: err,82 cause: err,
90 }83 }
internal/s3select/progress.go+24−7
@@ -25,7 +25,10 @@ import ( · 24 unmodified lines
25 "sync"25 "sync"
26 "sync/atomic"26 "sync/atomic"
27 27
28 "github.com/klauspost/compress/s2"
29 "github.com/klauspost/compress/zstd"
28 gzip "github.com/klauspost/pgzip"30 gzip "github.com/klauspost/pgzip"
31 "github.com/pierrec/lz4"
29)32)
30 33
31type countUpReader struct {34type countUpReader struct {
@@ -58,7 +61,7 @@ type progressReader struct { · 26 unmodified lines
58 processedReader *countUpReader61 processedReader *countUpReader
59 62
60 closedMu sync.Mutex63 closedMu sync.Mutex
61 gzr *gzip.Reader64 closer io.ReadCloser
62 closed bool65 closed bool
63}66}
64 67
@@ -80,8 +83,8 @@ func (pr *progressReader) Close() error { · 15 unmodified lines
80 return nil83 return nil
81 }84 }
82 pr.closed = true85 pr.closed = true
83 if pr.gzr != nil {86 if pr.closer != nil {
84 pr.gzr.Close()87 pr.closer.Close()
85 }88 }
86 return pr.rc.Close()89 return pr.rc.Close()
87}90}
@@ -102,23 +105,37 @@ func newProgressReader(rc io.ReadCloser, compType CompressionType) (*progressRea · 14 unmodified lines
102 rc: rc,105 rc: rc,
103 scannedReader: scannedReader,106 scannedReader: scannedReader,
104 }107 }
105 var err error
106 var r io.Reader108 var r io.Reader
107 109
108 switch compType {110 switch compType {
109 case noneType:111 case noneType:
110 r = scannedReader112 r = scannedReader
111 case gzipType:113 case gzipType:
112 pr.gzr, err = gzip.NewReader(scannedReader)114 gzr, err := gzip.NewReader(scannedReader)
113 if err != nil {115 if err != nil {
114 if errors.Is(err, gzip.ErrHeader) || errors.Is(err, gzip.ErrChecksum) {116 if errors.Is(err, gzip.ErrHeader) || errors.Is(err, gzip.ErrChecksum) {
115 return nil, errInvalidGZIPCompressionFormat(err)117 return nil, errInvalidCompression(err, compType)
116 }118 }
117 return nil, errTruncatedInput(err)119 return nil, errTruncatedInput(err)
118 }120 }
119 r = pr.gzr121 r = gzr
122 pr.closer = gzr
120 case bzip2Type:123 case bzip2Type:
121 r = bzip2.NewReader(scannedReader)124 r = bzip2.NewReader(scannedReader)
125 case zstdType:
126 // Set a max window of 64MB. More than reasonable.
127 zr, err := zstd.NewReader(scannedReader, zstd.WithDecoderConcurrency(2), zstd.WithDecoderMaxWindow(64<<20))
128 if err != nil {
129 return nil, errInvalidCompression(err, compType)
130 }
131 r = zr
132 pr.closer = zr.IOReadCloser()
133 case lz4Type:
134 r = lz4.NewReader(scannedReader)
135 case s2Type:
136 r = s2.NewReader(scannedReader)
137 case snappyType:
138 r = s2.NewReader(scannedReader, s2.ReaderMaxBlockSize(64<<10))
122 default:139 default:
123 return nil, errInvalidCompressionFormat(fmt.Errorf("unknown compression type '%v'", compType))140 return nil, errInvalidCompressionFormat(fmt.Errorf("unknown compression type '%v'", compType))
124 }141 }
internal/s3select/select.go+28−7
@@ -31,12 +31,16 @@ import ( · 30 unmodified lines
31 "strings"31 "strings"
32 "sync"32 "sync"
33 33
34 "github.com/klauspost/compress/s2"
35 "github.com/klauspost/compress/zstd"
36 gzip "github.com/klauspost/pgzip"
34 "github.com/minio/minio/internal/s3select/csv"37 "github.com/minio/minio/internal/s3select/csv"
35 "github.com/minio/minio/internal/s3select/json"38 "github.com/minio/minio/internal/s3select/json"
36 "github.com/minio/minio/internal/s3select/parquet"39 "github.com/minio/minio/internal/s3select/parquet"
37 "github.com/minio/minio/internal/s3select/simdj"40 "github.com/minio/minio/internal/s3select/simdj"
38 "github.com/minio/minio/internal/s3select/sql"41 "github.com/minio/minio/internal/s3select/sql"
39 "github.com/minio/simdjson-go"42 "github.com/minio/simdjson-go"
43 "github.com/pierrec/lz4"
40)44)
41 45
42type recordReader interface {46type recordReader interface {
@@ -57,8 +61,13 @@ type CompressionType string · 14 unmodified lines
57 61
58const (62const (
59 noneType CompressionType = "none"63 noneType CompressionType = "none"
60 gzipType CompressionType = "gzip"64 gzipType CompressionType = "GZIP"
61 bzip2Type CompressionType = "bzip2"65 bzip2Type CompressionType = "BZIP2"
66
67 zstdType CompressionType = "ZSTD"
68 lz4Type CompressionType = "LZ4"
69 s2Type CompressionType = "S2"
70 snappyType CompressionType = "SNAPPY"
62)71)
63 72
64const (73const (
@@ -87,13 +96,13 @@ func (c *CompressionType) UnmarshalXML(d *xml.Decoder, start xml.StartElement) e · 22 unmodified lines
87 return errMalformedXML(err)96 return errMalformedXML(err)
88 }97 }
89 98
90 parsedType := CompressionType(strings.ToLower(s))99 parsedType := CompressionType(strings.ToUpper(s))
91 if s == "" {100 if s == "" || parsedType == "NONE" {
92 parsedType = noneType101 parsedType = noneType
93 }102 }
94 103
95 switch parsedType {104 switch parsedType {
96 case noneType, gzipType, bzip2Type:105 case noneType, gzipType, bzip2Type, snappyType, s2Type, zstdType, lz4Type:
97 default:106 default:
98 return errInvalidCompressionFormat(fmt.Errorf("invalid compression format '%v'", s))107 return errInvalidCompressionFormat(fmt.Errorf("invalid compression format '%v'", s))
99 }108 }
@@ -127,7 +136,7 @@ func (input *InputSerialization) UnmarshalXML(d *xml.Decoder, start xml.StartEle · 27 unmodified lines
127 }136 }
128 137
129 // If no compression is specified, set to noneType138 // If no compression is specified, set to noneType
130 if parsedInput.CompressionType == CompressionType("") {139 if parsedInput.CompressionType == "" {
131 parsedInput.CompressionType = noneType140 parsedInput.CompressionType = noneType
132 }141 }
133 142
@@ -309,7 +318,19 @@ func (s3Select *S3Select) Open(getReader func(offset, length int64) (io.ReadClos · 175 unmodified lines
309 rc.Close()318 rc.Close()
310 var stErr bzip2.StructuralError319 var stErr bzip2.StructuralError
311 if errors.As(err, &stErr) {320 if errors.As(err, &stErr) {
312 return errInvalidBZIP2CompressionFormat(err)321 return errInvalidCompression(err, s3Select.Input.CompressionType)
322 }
323 // Test these compressor errors
324 errs := []error{
325 gzip.ErrHeader, gzip.ErrChecksum,
326 s2.ErrCorrupt, s2.ErrUnsupported, s2.ErrCRC,
327 zstd.ErrBlockTooSmall, zstd.ErrMagicMismatch, zstd.ErrWindowSizeExceeded, zstd.ErrUnknownDictionary, zstd.ErrWindowSizeTooSmall,
328 lz4.ErrInvalid, lz4.ErrBlockDependency,
329 }
330 for _, e := range errs {
331 if errors.Is(err, e) {
332 return errInvalidCompression(err, s3Select.Input.CompressionType)
333 }
313 }334 }
314 return err335 return err
315 }336 }

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t