// Copyright (c) 2015-2021 MinIO, Inc.//// This file is part of MinIO Object Storage stack//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as published by// the Free Software Foundation, either version 3 of the License, or// (at your option) any later version.//// This program is distributed in the hope that it will be useful// but WITHOUT ANY WARRANTY; without even the implied warranty of// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the// GNU Affero General Public License for more details.//// You should have received a copy of the GNU Affero General Public License// along with this program.  If not, see <http://www.gnu.org/licenses/>. package s3select import (	"bytes"	"encoding/xml"	"fmt"	"io"	"io/ioutil"	"net/http"	"os"	"reflect"	"strings"	"testing" 	"github.com/klauspost/cpuid/v2"	"github.com/minio/minio-go/v7"	"github.com/minio/simdjson-go") func newStringRSC(s string) io.ReadSeekCloser {	return newBytesRSC([]byte(s))} func newBytesRSC(b []byte) io.ReadSeekCloser {	r := bytes.NewReader(b)	segmentReader := func(offset int64) (io.ReadCloser, error) {		_, err := r.Seek(offset, io.SeekStart)		if err != nil {			return nil, err		}		return io.NopCloser(r), nil	}	return NewObjectReadSeekCloser(segmentReader, int64(len(b)))} type testResponseWriter struct {	statusCode int	response   []byte} func (w *testResponseWriter) Header() http.Header {	return nil} func (w *testResponseWriter) Write(p []byte) (int, error) {	w.response = append(w.response, p...)	return len(p), nil} func (w *testResponseWriter) WriteHeader(statusCode int) {	w.statusCode = statusCode} func (w *testResponseWriter) Flush() {} func TestJSONQueries(t *testing.T) {	input := `{"id": 0,"title": "Test Record","desc": "Some text","synonyms": ["foo", "bar", "whatever"]}	{"id": 1,"title": "Second Record","desc": "another text","synonyms": ["some", "synonym", "value"]}	{"id": 2,"title": "Second Record","desc": "another text","numbers": [2, 3.0, 4]}	{"id": 3,"title": "Second Record","desc": "another text","nested": [[2, 3.0, 4], [7, 8.5, 9]]}` 	testTable := []struct {		name       string		query      string		requestXML []byte // override request XML		wantResult string		withJSON   string // Override JSON input	}{		{			name:       "select-in-array-full",			query:      `SELECT * from s3object s WHERE 'bar' IN s.synonyms[*]`,			wantResult: `{"id":0,"title":"Test Record","desc":"Some text","synonyms":["foo","bar","whatever"]}`,		},		{			name:  "simple-in-array",			query: `SELECT * from s3object s WHERE s.id IN (1,3)`,			wantResult: `{"id":1,"title":"Second Record","desc":"another text","synonyms":["some","synonym","value"]}{"id":3,"title":"Second Record","desc":"another text","nested":[[2,3,4],[7,8.5,9]]}`,		},		{			name:       "select-in-array-single",			query:      `SELECT synonyms from s3object s WHERE 'bar' IN s.synonyms[*] `,			wantResult: `{"synonyms":["foo","bar","whatever"]}`,		},		{			name:       "donatello-1",			query:      `SELECT * from s3object s WHERE 'bar' in s.synonyms`,			wantResult: `{"id":0,"title":"Test Record","desc":"Some text","synonyms":["foo","bar","whatever"]}`,		},		{			name:       "donatello-2",			query:      `SELECT * from s3object s WHERE 'bar' in s.synonyms[*]`,			wantResult: `{"id":0,"title":"Test Record","desc":"Some text","synonyms":["foo","bar","whatever"]}`,		},		{			name:  "bignum-1",			query: `SELECT id from s3object s WHERE s.id <= 9223372036854775807`,			wantResult: `{"id":0}{"id":1}{"id":2}{"id":3}`,		},		{			name:  "bignum-2",			query: `SELECT id from s3object s WHERE s.id >= -9223372036854775808`,			wantResult: `{"id":0}{"id":1}{"id":2}{"id":3}`,		},		{			name:       "donatello-3",			query:      `SELECT * from s3object s WHERE 'value' IN s.synonyms[*]`,			wantResult: `{"id":1,"title":"Second Record","desc":"another text","synonyms":["some","synonym","value"]}`,		},		{			name:       "select-in-number",			query:      `SELECT * from s3object s WHERE 4 in s.numbers[*]`,			wantResult: `{"id":2,"title":"Second Record","desc":"another text","numbers":[2,3,4]}`,		},		{			name:       "select-in-number-float",			query:      `SELECT * from s3object s WHERE 3 in s.numbers[*]`,			wantResult: `{"id":2,"title":"Second Record","desc":"another text","numbers":[2,3,4]}`,		},		{			name:       "select-in-number-float-in-sql",			query:      `SELECT * from s3object s WHERE 3.0 in s.numbers[*]`,			wantResult: `{"id":2,"title":"Second Record","desc":"another text","numbers":[2,3,4]}`,		},		{			name:       "select-in-list-match",			query:      `SELECT * from s3object s WHERE (2,3,4) IN s.nested[*]`,			wantResult: `{"id":3,"title":"Second Record","desc":"another text","nested":[[2,3,4],[7,8.5,9]]}`,		},		{			name:       "select-in-nested-float",			query:      `SELECT s.nested from s3object s WHERE 8.5 IN s.nested[*][*]`,			wantResult: `{"nested":[[2,3,4],[7,8.5,9]]}`,		},		{			name:       "select-in-combine-and",			query:      `SELECT s.nested from s3object s WHERE (8.5 IN s.nested[*][*]) AND (s.id > 0)`,			wantResult: `{"nested":[[2,3,4],[7,8.5,9]]}`,		},		{			name:       "select-in-combine-and-no",			query:      `SELECT s.nested from s3object s WHERE (8.5 IN s.nested[*][*]) AND (s.id = 0)`,			wantResult: ``,		},		{			name:       "select-in-nested-float-no-flat",			query:      `SELECT s.nested from s3object s WHERE 8.5 IN s.nested[*]`,			wantResult: ``,		},		{			name:       "select-empty-field-result",			query:      `SELECT * from s3object s WHERE s.nested[0][0] = 2`,			wantResult: `{"id":3,"title":"Second Record","desc":"another text","nested":[[2,3,4],[7,8.5,9]]}`,		},		{			name:       "select-arrays-specific",			query:      `SELECT * from s3object s WHERE s.nested[1][0] = 7`,			wantResult: `{"id":3,"title":"Second Record","desc":"another text","nested":[[2,3,4],[7,8.5,9]]}`,		},		{			name:       "wrong-index-no-result",			query:      `SELECT * from s3object s WHERE s.nested[0][0] = 7`,			wantResult: ``,		},		{			name:  "not-equal-result",			query: `SELECT * from s3object s WHERE s.nested[1][0] != 7`,			wantResult: `{"id":0,"title":"Test Record","desc":"Some text","synonyms":["foo","bar","whatever"]}{"id":1,"title":"Second Record","desc":"another text","synonyms":["some","synonym","value"]}{"id":2,"title":"Second Record","desc":"another text","numbers":[2,3,4]}`,		},		{			name:       "indexed-list-match",			query:      `SELECT * from s3object s WHERE (7,8.5,9) IN s.nested[1]`,			wantResult: ``,		},		{			name:       "indexed-list-match-equals",			query:      `SELECT * from s3object s WHERE (7,8.5,9) = s.nested[1]`,			wantResult: `{"id":3,"title":"Second Record","desc":"another text","nested":[[2,3,4],[7,8.5,9]]}`,		},		{			name:       "indexed-list-match-equals-s-star",			query:      `SELECT s.* from s3object s WHERE (7,8.5,9) = s.nested[1]`,			wantResult: `{"id":3,"title":"Second Record","desc":"another text","nested":[[2,3,4],[7,8.5,9]]}`,		},		{			name:       "indexed-list-match-equals-s-index",			query:      `SELECT s.nested[1], s.nested[0] from s3object s WHERE (7,8.5,9) = s.nested[1]`,			wantResult: `{"_1":[7,8.5,9],"_2":[2,3,4]}`,		},		{			name:  "indexed-list-match-not-equals",			query: `SELECT * from s3object s WHERE (7,8.5,9) != s.nested[1]`,			wantResult: `{"id":0,"title":"Test Record","desc":"Some text","synonyms":["foo","bar","whatever"]}{"id":1,"title":"Second Record","desc":"another text","synonyms":["some","synonym","value"]}{"id":2,"title":"Second Record","desc":"another text","numbers":[2,3,4]}`,		},		{			name:       "indexed-list-square-bracket",			query:      `SELECT * from s3object s WHERE [7,8.5,9] = s.nested[1]`,			wantResult: `{"id":3,"title":"Second Record","desc":"another text","nested":[[2,3,4],[7,8.5,9]]}`,		},		{			name:       "indexed-list-square-bracket",			query:      `SELECT * from s3object s WHERE [7,8.5,9] IN s.nested`,			wantResult: `{"id":3,"title":"Second Record","desc":"another text","nested":[[2,3,4],[7,8.5,9]]}`,		},		{			name:  "indexed-list-square-bracket",			query: `SELECT * from s3object s WHERE id IN [3,2]`,			wantResult: `{"id":2,"title":"Second Record","desc":"another text","numbers":[2,3,4]}{"id":3,"title":"Second Record","desc":"another text","nested":[[2,3,4],[7,8.5,9]]}`,		},		{			name:       "index-wildcard-in",			query:      `SELECT * from s3object s WHERE (8.5) IN s.nested[1][*]`,			wantResult: `{"id":3,"title":"Second Record","desc":"another text","nested":[[2,3,4],[7,8.5,9]]}`,		},		{			name:       "index-wildcard-in",			query:      `SELECT * from s3object s WHERE (8.0+0.5) IN s.nested[1][*]`,			wantResult: `{"id":3,"title":"Second Record","desc":"another text","nested":[[2,3,4],[7,8.5,9]]}`,		},		{			name:       "compare-mixed",			query:      `SELECT id from s3object s WHERE value = true`,			wantResult: `{"id":1}`,			withJSON: `{"id":0, "value": false}{"id":1, "value": true}{"id":2, "value": 42}{"id":3, "value": "true"}`,		},		{			name:       "compare-mixed-not",			query:      `SELECT COUNT(id) as n from s3object s WHERE value != true`,			wantResult: `{"n":3}`,			withJSON: `{"id":0, "value": false}{"id":1, "value": true}{"id":2, "value": 42}{"id":3, "value": "true"}`,		},		{			name:       "index-wildcard-in",			query:      `SELECT * from s3object s WHERE title = 'Test Record'`,			wantResult: `{"id":0,"title":"Test Record","desc":"Some text","synonyms":["foo","bar","whatever"]}`,		},		{			name: "select-output-field-as-csv",			requestXML: []byte(`<?xml version="1.0" encoding="UTF-8"?><SelectObjectContentRequest>    <Expression>SELECT s.synonyms from s3object s WHERE 'whatever' IN s.synonyms</Expression>    <ExpressionType>SQL</ExpressionType>    <InputSerialization>        <CompressionType>NONE</CompressionType>        <JSON>            <Type>DOCUMENT</Type>        </JSON>    </InputSerialization>    <OutputSerialization>        <CSV>	   <QuoteCharacter>"</QuoteCharacter>        </CSV>    </OutputSerialization>    <RequestProgress>        <Enabled>FALSE</Enabled>    </RequestProgress></SelectObjectContentRequest>`),			wantResult: `"[""foo"",""bar"",""whatever""]"`,		},		{			name:  "document",			query: "",			requestXML: []byte(`<?xml version="1.0" encoding="UTF-8"?><SelectObjectContentRequest>    <Expression>select * from s3object[*].elements[*] s where s.element_type = '__elem__merfu'</Expression>    <ExpressionType>SQL</ExpressionType>    <InputSerialization>        <CompressionType>NONE</CompressionType>        <JSON>            <Type>DOCUMENT</Type>        </JSON>    </InputSerialization>    <OutputSerialization>        <JSON>        </JSON>    </OutputSerialization>    <RequestProgress>        <Enabled>FALSE</Enabled>    </RequestProgress></SelectObjectContentRequest>`),			withJSON: `{  "name": "small_pdf1.pdf",  "lume_id": "9507193e-572d-4f95-bcf1-e9226d96be65",  "elements": [    {      "element_type": "__elem__image",      "element_id": "859d09c4-7cf1-4a37-9674-3a7de8b56abc",      "attributes": {        "__attr__image_dpi": 300,        "__attr__image_size": [          2550,          3299        ],        "__attr__image_index": 1,        "__attr__image_format": "JPEG",        "__attr__file_extension": "jpg",        "__attr__data": null      }    },    {      "element_type": "__elem__merfu",      "element_id": "d868aefe-ef9a-4be2-b9b2-c9fd89cc43eb",      "attributes": {        "__attr__image_dpi": 300,        "__attr__image_size": [          2550,          3299        ],        "__attr__image_index": 2,        "__attr__image_format": "JPEG",        "__attr__file_extension": "jpg",        "__attr__data": null      }    }  ],  "data": "asdascasdc1234e123erdasdas"}`,			wantResult: `{"element_type":"__elem__merfu","element_id":"d868aefe-ef9a-4be2-b9b2-c9fd89cc43eb","attributes":{"__attr__image_dpi":300,"__attr__image_size":[2550,3299],"__attr__image_index":2,"__attr__image_format":"JPEG","__attr__file_extension":"jpg","__attr__data":null}}`,		},		{			name:       "date_diff_month",			query:      `SELECT date_diff(MONTH, '2019-10-20T', '2020-01-20T') FROM S3Object LIMIT 1`,			wantResult: `{"_1":3}`,		},		{			name:       "date_diff_month_neg",			query:      `SELECT date_diff(MONTH, '2020-01-20T', '2019-10-20T') FROM S3Object LIMIT 1`,			wantResult: `{"_1":-3}`,		},		// Examples from https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-glacier-select-sql-reference-date.html#s3-glacier-select-sql-reference-date-diff		{			name:       "date_diff_year",			query:      `SELECT date_diff(year, '2010-01-01T', '2011-01-01T') FROM S3Object LIMIT 1`,			wantResult: `{"_1":1}`,		},		{			name:       "date_diff_year",			query:      `SELECT date_diff(month, '2010-01-01T', '2010-05T') FROM S3Object LIMIT 1`,			wantResult: `{"_1":4}`,		},		{			name:       "date_diff_month_oney",			query:      `SELECT date_diff(month, '2010T', '2011T') FROM S3Object LIMIT 1`,			wantResult: `{"_1":12}`,		},		{			name:       "date_diff_month_neg",			query:      `SELECT date_diff(month, '2011T', '2010T') FROM S3Object LIMIT 1`,			wantResult: `{"_1":-12}`,		},		{			name:       "date_diff_days",			query:      `SELECT date_diff(day, '2010-01-01T23:00:00Z', '2010-01-02T01:00:00Z') FROM S3Object LIMIT 1`,			wantResult: `{"_1":0}`,		},		{			name:       "date_diff_days_one",			query:      `SELECT date_diff(day, '2010-01-01T23:00:00Z', '2010-01-02T23:00:00Z') FROM S3Object LIMIT 1`,			wantResult: `{"_1":1}`,		},		{			name:       "cast_from_int_to_float",			query:      `SELECT cast(1 as float) FROM S3Object LIMIT 1`,			wantResult: `{"_1":1}`,		},		{			name:       "cast_from_float_to_float",			query:      `SELECT cast(1.0 as float) FROM S3Object LIMIT 1`,			wantResult: `{"_1":1}`,		},		{			name:       "arithmetic_integer_operand",			query:      `SELECT 1 / 2 FROM S3Object LIMIT 1`,			wantResult: `{"_1":0}`,		},		{			name:       "arithmetic_float_operand",			query:      `SELECT 1.0 / 2.0 * .3 FROM S3Object LIMIT 1`,			wantResult: `{"_1":0.15}`,		},		{			name:       "arithmetic_integer_float_operand",			query:      `SELECT 3.0 / 2, 5 / 2.0 FROM S3Object LIMIT 1`,			wantResult: `{"_1":1.5,"_2":2.5}`,		},		{			name:  "limit-1",			query: `SELECT * FROM S3Object[*].elements[*] LIMIT 1`,			requestXML: []byte(`<?xml version="1.0" encoding="UTF-8"?><SelectObjectContentRequest>    <Expression>select * from s3object[*].elements[*] s where s.element_type = '__elem__merfu'</Expression>    <ExpressionType>SQL</ExpressionType>    <InputSerialization>        <CompressionType>NONE</CompressionType>        <JSON>            <Type>DOCUMENT</Type>        </JSON>    </InputSerialization>    <OutputSerialization>        <JSON>        </JSON>    </OutputSerialization>    <RequestProgress>        <Enabled>FALSE</Enabled>    </RequestProgress></SelectObjectContentRequest>`),			wantResult: `{"element_type":"__elem__merfu","element_id":"d868aefe-ef9a-4be2-b9b2-c9fd89cc43eb","attributes":{"__attr__image_dpi":300,"__attr__image_size":[2550,3299],"__attr__image_index":2,"__attr__image_format":"JPEG","__attr__file_extension":"jpg","__attr__data":null}}`,			withJSON: `{  "name": "small_pdf1.pdf",  "lume_id": "9507193e-572d-4f95-bcf1-e9226d96be65",  "elements": [    {      "element_type": "__elem__image",      "element_id": "859d09c4-7cf1-4a37-9674-3a7de8b56abc",      "attributes": {        "__attr__image_dpi": 300,        "__attr__image_size": [          2550,          3299        ],        "__attr__image_index": 1,        "__attr__image_format": "JPEG",        "__attr__file_extension": "jpg",        "__attr__data": null      }    },    {      "element_type": "__elem__merfu",      "element_id": "d868aefe-ef9a-4be2-b9b2-c9fd89cc43eb",      "attributes": {        "__attr__image_dpi": 300,        "__attr__image_size": [          2550,          3299        ],        "__attr__image_index": 2,        "__attr__image_format": "JPEG",        "__attr__file_extension": "jpg",        "__attr__data": null      }    }  ],  "data": "asdascasdc1234e123erdasdas"}`,		},		{			name:       "limit-2",			query:      `select * from s3object[*].person[*] limit 1`,			wantResult: `{"Id":1,"Name":"Anshu","Address":"Templestowe","Car":"Jeep"}`,			withJSON:   `{ "person": [ { "Id": 1, "Name": "Anshu", "Address": "Templestowe", "Car": "Jeep" }, { "Id": 2, "Name": "Ben Mostafa", "Address": "Las Vegas", "Car": "Mustang" }, { "Id": 3, "Name": "Rohan Wood", "Address": "Wooddon", "Car": "VW" } ] }`,		},		{			name:       "lower-case-is",			query:      `select * from s3object[*] as s where s.request.header['User-Agent'] is not null`,			wantResult: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}`,			withJSON: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}{"request":{"uri":"/2","header":{}}}`,		},		{

Lines 1–500 of 1958. Load more