select: Add more compression formats (#13142)

Support Zstandard, LZ4, S2, and snappy as additional compression formats for S3 Select.

Klaus Post committed 5 years ago

Commit b2c92cd1 parent 42b1d926 files · +62 −30Browse files

Unified · Split

6 files

docs/select/README.md +2 −1
go.mod +1 −1
go.sum +2 −2
internal/s3select/errors.go +5 −12
internal/s3select/progress.go +24 −7
internal/s3select/select.go +28 −7

@@ -5,7 +5,8 @@ You can use the Select API to query objects with following features: · 4 unmodified lines
5	5
6	6	- Objects must be in CSV, JSON, or Parquet(*) format.
7	7	- UTF-8 is the only encoding type the Select API supports.
8		−- GZIP or BZIP2 - CSV and JSON files can be compressed using GZIP or BZIP2. The Select API supports columnar compression for Parquet using GZIP, Snappy, LZ4. Whole object compression is not supported for Parquet objects.
	8	+- GZIP or BZIP2 - CSV and JSON files can be compressed using GZIP, BZIP2, [ZSTD](https://facebook.github.io/zstd/), and streaming formats of [LZ4](https://lz4.github.io/lz4/), [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) and [SNAPPY](http://google.github.io/snappy/).
	9	+- Parquet API supports columnar compression for using GZIP, Snappy, LZ4. Whole object compression is not supported for Parquet objects.
9	10	- Server-side encryption - The Select API supports querying objects that are protected with server-side encryption.
10	11
11	12	Type inference and automatic conversion of values is performed based on the context when the value is un-typed (such as when reading CSV data). If present, the CAST function overrides automatic conversion.

go.mod+1−1

@@ -32,7 +32,7 @@ require ( · 31 unmodified lines
32	32	github.com/gorilla/mux v1.8.0
33	33	github.com/jcmturner/gokrb5/v8 v8.4.2
34	34	github.com/json-iterator/go v1.1.11
35		− github.com/klauspost/compress v1.13.4
	35	+ github.com/klauspost/compress v1.13.5
36	36	github.com/klauspost/cpuid/v2 v2.0.4
37	37	github.com/klauspost/pgzip v1.2.5
38	38	github.com/klauspost/readahead v1.3.1

go.sum+2−2

@@ -883,8 +883,8 @@ github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs · 882 unmodified lines
883	883	github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
884	884	github.com/klauspost/compress v1.11.12/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
885	885	github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
886		−github.com/klauspost/compress v1.13.4 h1:0zhec2I8zGnjWcKyLl6i3gPqKANCCn5e9xmviEEeX6s=
887		−github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
	886	+github.com/klauspost/compress v1.13.5 h1:9O69jUPDcsT9fEm74W92rZL9FQY7rCdaXVneq+yyzl4=
	887	+github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
888	888	github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
889	889	github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
890	890	github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=

internal/s3select/errors.go+5−12

@@ -17,6 +17,8 @@ · 16 unmodified lines
17	17
18	18	package s3select
19	19
	20	+import "strings"
	21	+
20	22	// SelectError - represents s3 select error specified in
21	23	// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html#RESTObjectSELECTContent-responses-special-errors.
22	24	type SelectError interface {
@@ -66,25 +68,16 @@ func errMalformedXML(err error) *s3Error { · 43 unmodified lines
66	68	func errInvalidCompressionFormat(err error) *s3Error {
67	69	return &s3Error{
68	70	code: "InvalidCompressionFormat",
69		− message: "The file is not in a supported compression format. Only GZIP and BZIP2 are supported.",
70		− statusCode: 400,
71		− cause: err,
72		− }
73		−}
74		−
75		−func errInvalidBZIP2CompressionFormat(err error) *s3Error {
76		− return &s3Error{
77		− code: "InvalidCompressionFormat",
78		− message: "BZIP2 is not applicable to the queried object. Please correct the request and try again.",
	71	+ message: "The file is not in a supported compression format. GZIP, BZIP2, ZSTD, LZ4, S2 and SNAPPY are supported.",
79	72	statusCode: 400,
80	73	cause: err,
81	74	}
82	75	}
83	76
84		−func errInvalidGZIPCompressionFormat(err error) *s3Error {
	77	+func errInvalidCompression(err error, t CompressionType) *s3Error {
85	78	return &s3Error{
86	79	code: "InvalidCompressionFormat",
87		− message: "GZIP is not applicable to the queried object. Please correct the request and try again.",
	80	+ message: strings.ToUpper(string(t)) + " is not applicable to the queried object. Please correct the request and try again.",
88	81	statusCode: 400,
89	82	cause: err,
90	83	}

internal/s3select/progress.go+24−7

@@ -25,7 +25,10 @@ import ( · 24 unmodified lines
25	25	"sync"
26	26	"sync/atomic"
27	27
	28	+ "github.com/klauspost/compress/s2"
	29	+ "github.com/klauspost/compress/zstd"
28	30	gzip "github.com/klauspost/pgzip"
	31	+ "github.com/pierrec/lz4"
29	32	)
30	33
31	34	type countUpReader struct {
@@ -58,7 +61,7 @@ type progressReader struct { · 26 unmodified lines
58	61	processedReader *countUpReader
59	62
60	63	closedMu sync.Mutex
61		− gzr *gzip.Reader
	64	+ closer io.ReadCloser
62	65	closed bool
63	66	}
64	67
@@ -80,8 +83,8 @@ func (pr *progressReader) Close() error { · 15 unmodified lines
80	83	return nil
81	84	}
82	85	pr.closed = true
83		− if pr.gzr != nil {
84		− pr.gzr.Close()
	86	+ if pr.closer != nil {
	87	+ pr.closer.Close()
85	88	}
86	89	return pr.rc.Close()
87	90	}
@@ -102,23 +105,37 @@ func newProgressReader(rc io.ReadCloser, compType CompressionType) (*progressRea · 14 unmodified lines
102	105	rc: rc,
103	106	scannedReader: scannedReader,
104	107	}
105		− var err error
106	108	var r io.Reader
107	109
108	110	switch compType {
109	111	case noneType:
110	112	r = scannedReader
111	113	case gzipType:
112		− pr.gzr, err = gzip.NewReader(scannedReader)
	114	+ gzr, err := gzip.NewReader(scannedReader)
113	115	if err != nil {
114	116	if errors.Is(err, gzip.ErrHeader) \|\| errors.Is(err, gzip.ErrChecksum) {
115		− return nil, errInvalidGZIPCompressionFormat(err)
	117	+ return nil, errInvalidCompression(err, compType)
116	118	}
117	119	return nil, errTruncatedInput(err)
118	120	}
119		− r = pr.gzr
	121	+ r = gzr
	122	+ pr.closer = gzr
120	123	case bzip2Type:
121	124	r = bzip2.NewReader(scannedReader)
	125	+ case zstdType:
	126	+ // Set a max window of 64MB. More than reasonable.
	127	+ zr, err := zstd.NewReader(scannedReader, zstd.WithDecoderConcurrency(2), zstd.WithDecoderMaxWindow(64<<20))
	128	+ if err != nil {
	129	+ return nil, errInvalidCompression(err, compType)
	130	+ }
	131	+ r = zr
	132	+ pr.closer = zr.IOReadCloser()
	133	+ case lz4Type:
	134	+ r = lz4.NewReader(scannedReader)
	135	+ case s2Type:
	136	+ r = s2.NewReader(scannedReader)
	137	+ case snappyType:
	138	+ r = s2.NewReader(scannedReader, s2.ReaderMaxBlockSize(64<<10))
122	139	default:
123	140	return nil, errInvalidCompressionFormat(fmt.Errorf("unknown compression type '%v'", compType))
124	141	}

internal/s3select/select.go+28−7

@@ -31,12 +31,16 @@ import ( · 30 unmodified lines
31	31	"strings"
32	32	"sync"
33	33
	34	+ "github.com/klauspost/compress/s2"
	35	+ "github.com/klauspost/compress/zstd"
	36	+ gzip "github.com/klauspost/pgzip"
34	37	"github.com/minio/minio/internal/s3select/csv"
35	38	"github.com/minio/minio/internal/s3select/json"
36	39	"github.com/minio/minio/internal/s3select/parquet"
37	40	"github.com/minio/minio/internal/s3select/simdj"
38	41	"github.com/minio/minio/internal/s3select/sql"
39	42	"github.com/minio/simdjson-go"
	43	+ "github.com/pierrec/lz4"
40	44	)
41	45
42	46	type recordReader interface {
@@ -57,8 +61,13 @@ type CompressionType string · 14 unmodified lines
57	61
58	62	const (
59	63	noneType CompressionType = "none"
60		− gzipType CompressionType = "gzip"
61		− bzip2Type CompressionType = "bzip2"
	64	+ gzipType CompressionType = "GZIP"
	65	+ bzip2Type CompressionType = "BZIP2"
	66	+
	67	+ zstdType CompressionType = "ZSTD"
	68	+ lz4Type CompressionType = "LZ4"
	69	+ s2Type CompressionType = "S2"
	70	+ snappyType CompressionType = "SNAPPY"
62	71	)
63	72
64	73	const (
@@ -87,13 +96,13 @@ func (c CompressionType) UnmarshalXML(d xml.Decoder, start xml.StartElement) e · 22 unmodified lines
87	96	return errMalformedXML(err)
88	97	}
89	98
90		− parsedType := CompressionType(strings.ToLower(s))
91		− if s == "" {
	99	+ parsedType := CompressionType(strings.ToUpper(s))
	100	+ if s == "" \|\| parsedType == "NONE" {
92	101	parsedType = noneType
93	102	}
94	103
95	104	switch parsedType {
96		− case noneType, gzipType, bzip2Type:
	105	+ case noneType, gzipType, bzip2Type, snappyType, s2Type, zstdType, lz4Type:
97	106	default:
98	107	return errInvalidCompressionFormat(fmt.Errorf("invalid compression format '%v'", s))
99	108	}
@@ -127,7 +136,7 @@ func (input InputSerialization) UnmarshalXML(d xml.Decoder, start xml.StartEle · 27 unmodified lines
127	136	}
128	137
129	138	// If no compression is specified, set to noneType
130		− if parsedInput.CompressionType == CompressionType("") {
	139	+ if parsedInput.CompressionType == "" {
131	140	parsedInput.CompressionType = noneType
132	141	}
133	142
@@ -309,7 +318,19 @@ func (s3Select *S3Select) Open(getReader func(offset, length int64) (io.ReadClos · 175 unmodified lines
309	318	rc.Close()
310	319	var stErr bzip2.StructuralError
311	320	if errors.As(err, &stErr) {
312		− return errInvalidBZIP2CompressionFormat(err)
	321	+ return errInvalidCompression(err, s3Select.Input.CompressionType)
	322	+ }
	323	+ // Test these compressor errors
	324	+ errs := []error{
	325	+ gzip.ErrHeader, gzip.ErrChecksum,
	326	+ s2.ErrCorrupt, s2.ErrUnsupported, s2.ErrCRC,
	327	+ zstd.ErrBlockTooSmall, zstd.ErrMagicMismatch, zstd.ErrWindowSizeExceeded, zstd.ErrUnknownDictionary, zstd.ErrWindowSizeTooSmall,
	328	+ lz4.ErrInvalid, lz4.ErrBlockDependency,
	329	+ }
	330	+ for _, e := range errs {
	331	+ if errors.Is(err, e) {
	332	+ return errInvalidCompression(err, s3Select.Input.CompressionType)
	333	+ }
313	334	}
314	335	return err
315	336	}

select: Add more compression formats (#13142)

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code