S3 Select API Support for CSV (#6127)

Add support for trivial where clause cases

Arjun Mishra committed 8 years ago

Commit 7c14cdb1 parent 0e0232859 files · +30860 −3Browse files

Unified · Split

59 files

Makefile +2 −2
cmd/api-datatypes.go +81 −0
cmd/api-errors.go +679 −0
cmd/api-router.go +2 −0
cmd/generic-handlers.go +0 −1
cmd/object-handlers.go +187 −0
docs/select/README.md +98 −0
pkg/s3select/errors.go +486 −0
pkg/s3select/funcEval.go +231 −0
pkg/s3select/helpers.go +754 −0
pkg/s3select/input.go +381 −0
pkg/s3select/output.go +460 −0
pkg/s3select/select.go +415 −0
pkg/s3select/select_test.go +1070 −0
vendor/github.com/klauspost/compress/LICENSE +27 −0
vendor/github.com/klauspost/compress/flate/copy.go +32 −0
vendor/github.com/klauspost/compress/flate/crc32_amd64.go +41 −0
vendor/github.com/klauspost/compress/flate/crc32_amd64.s +213 −0
vendor/github.com/klauspost/compress/flate/crc32_noasm.go +35 −0
vendor/github.com/klauspost/compress/flate/deflate.go +1353 −0
vendor/github.com/klauspost/compress/flate/dict_decoder.go +184 −0
vendor/github.com/klauspost/compress/flate/gen.go +265 −0
vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go +701 −0
vendor/github.com/klauspost/compress/flate/huffman_code.go +344 −0
vendor/github.com/klauspost/compress/flate/inflate.go +868 −0
vendor/github.com/klauspost/compress/flate/reverse_bits.go +48 −0
vendor/github.com/klauspost/compress/flate/snappy.go +900 −0
vendor/github.com/klauspost/compress/flate/token.go +115 −0
vendor/github.com/klauspost/pgzip/GO_LICENSE +27 −0
vendor/github.com/klauspost/pgzip/LICENSE +22 −0
vendor/github.com/klauspost/pgzip/README.md +136 −0
vendor/github.com/klauspost/pgzip/circle.yml +7 −0
vendor/github.com/klauspost/pgzip/gunzip.go +573 −0
vendor/github.com/klauspost/pgzip/gzip.go +501 −0
vendor/github.com/xwb1989/sqlparser/CONTRIBUTORS.md +9 −0
vendor/github.com/xwb1989/sqlparser/LICENSE.md +201 −0
vendor/github.com/xwb1989/sqlparser/Makefile +22 −0
vendor/github.com/xwb1989/sqlparser/README.md +150 −0
vendor/github.com/xwb1989/sqlparser/analyzer.go +343 −0
vendor/github.com/xwb1989/sqlparser/ast.go +3450 −0
vendor/github.com/xwb1989/sqlparser/comments.go +293 −0
vendor/github.com/xwb1989/sqlparser/dependency/bytes2/buffer.go +65 −0
vendor/github.com/xwb1989/sqlparser/dependency/hack/hack.go +79 −0
vendor/github.com/xwb1989/sqlparser/dependency/querypb/query.pb.go +2734 −0
vendor/github.com/xwb1989/sqlparser/dependency/sqltypes/bind_variables.go +266 −0
vendor/github.com/xwb1989/sqlparser/dependency/sqltypes/plan_value.go +259 −0
vendor/github.com/xwb1989/sqlparser/dependency/sqltypes/testing.go +154 −0
vendor/github.com/xwb1989/sqlparser/dependency/sqltypes/type.go +288 −0
vendor/github.com/xwb1989/sqlparser/dependency/sqltypes/value.go +376 −0
vendor/github.com/xwb1989/sqlparser/encodable.go +99 −0
vendor/github.com/xwb1989/sqlparser/impossible_query.go +39 −0
vendor/github.com/xwb1989/sqlparser/normalizer.go +224 −0
vendor/github.com/xwb1989/sqlparser/parsed_query.go +119 −0
vendor/github.com/xwb1989/sqlparser/redact_query.go +19 −0
vendor/github.com/xwb1989/sqlparser/sql.go +6136 −0
vendor/github.com/xwb1989/sqlparser/sql.y +3159 −0
vendor/github.com/xwb1989/sqlparser/token.go +950 −0
vendor/github.com/xwb1989/sqlparser/tracked_buffer.go +140 −0
vendor/vendor.json +48 −0

@@ -42,8 +42,8 @@ ineffassign: · 41 unmodified lines
42		42
43	cyclo:	43	cyclo:
44	@echo "Running $@"	44	@echo "Running $@"
45	@${GOPATH}/bin/gocyclo -over 100 cmd	45	@${GOPATH}/bin/gocyclo -over 200 cmd
46	@${GOPATH}/bin/gocyclo -over 100 pkg	46	@${GOPATH}/bin/gocyclo -over 200 pkg
47		47
48	deadcode:	48	deadcode:
49	@echo "Running $@"	49	@echo "Running $@"

cmd/api-datatypes.go+81−0

@@ -25,6 +25,87 @@ const ( · 24 unmodified lines
25	responseRequestIDKey = "x-amz-request-id"	25	responseRequestIDKey = "x-amz-request-id"
26	)	26	)
27		27
		28	// CSVFileHeaderInfo -Can be either USE IGNORE OR NONE, defines what to do with
		29	// the first row
		30	type CSVFileHeaderInfo string
		31
		32	// Constants for file header info.
		33	const (
		34	CSVFileHeaderInfoNone CSVFileHeaderInfo = "NONE"
		35	CSVFileHeaderInfoIgnore = "IGNORE"
		36	CSVFileHeaderInfoUse = "USE"
		37	)
		38
		39	// SelectCompressionType - ONLY GZIP is supported
		40	type SelectCompressionType string
		41
		42	// Constants for compression types under select API.
		43	const (
		44	SelectCompressionNONE SelectCompressionType = "NONE"
		45	SelectCompressionGZIP = "GZIP"
		46	SelectCompressionBZIP = "BZIP2"
		47	)
		48
		49	// CSVQuoteFields - Can be either Always or AsNeeded
		50	type CSVQuoteFields string
		51
		52	// Constants for csv quote styles.
		53	const (
		54	CSVQuoteFieldsAlways CSVQuoteFields = "Always"
		55	CSVQuoteFieldsAsNeeded = "AsNeeded"
		56	)
		57
		58	// QueryExpressionType - Currently can only be SQL
		59	type QueryExpressionType string
		60
		61	// Constants for expression type.
		62	const (
		63	QueryExpressionTypeSQL QueryExpressionType = "SQL"
		64	)
		65
		66	// JSONType determines json input serialization type.
		67	type JSONType string
		68
		69	// Constants for JSONTypes.
		70	const (
		71	JSONDocumentType JSONType = "Document"
		72	JSONStreamType = "Stream"
		73	JSONLinesType = "Lines"
		74	)
		75
		76	// ObjectSelectRequest - represents the input select body
		77	type ObjectSelectRequest struct {
		78	XMLName xml.Name `xml:"SelectObjectContentRequest" json:"-"`
		79	Expression string
		80	ExpressionType QueryExpressionType
		81	InputSerialization struct {
		82	CompressionType SelectCompressionType
		83	CSV *struct {
		84	FileHeaderInfo CSVFileHeaderInfo
		85	RecordDelimiter string
		86	FieldDelimiter string
		87	QuoteCharacter string
		88	QuoteEscapeCharacter string
		89	Comments string
		90	}
		91	JSON *struct {
		92	Type JSONType
		93	}
		94	}
		95	OutputSerialization struct {
		96	CSV *struct {
		97	QuoteFields CSVQuoteFields
		98	RecordDelimiter string
		99	FieldDelimiter string
		100	QuoteCharacter string
		101	QuoteEscapeCharacter string
		102	}
		103	JSON *struct {
		104	RecordDelimiter string
		105	}
		106	}
		107	}
		108
28	// ObjectIdentifier carries key name for the object to delete.	109	// ObjectIdentifier carries key name for the object to delete.
29	type ObjectIdentifier struct {	110	type ObjectIdentifier struct {
30	ObjectName string `xml:"Key"`	111	ObjectName string `xml:"Key"`

cmd/api-errors.go+679−0

@@ -27,6 +27,7 @@ import ( · 26 unmodified lines
27	"github.com/minio/minio/pkg/dns"	27	"github.com/minio/minio/pkg/dns"
28	"github.com/minio/minio/pkg/event"	28	"github.com/minio/minio/pkg/event"
29	"github.com/minio/minio/pkg/hash"	29	"github.com/minio/minio/pkg/hash"
		30	"github.com/minio/minio/pkg/s3select"
30	)	31	)
31		32
32	// APIError structure	33	// APIError structure
@@ -190,6 +191,7 @@ const ( · 157 unmodified lines
190	ErrAdminCredentialsMismatch	191	ErrAdminCredentialsMismatch
191	ErrInsecureClientRequest	192	ErrInsecureClientRequest
192	ErrObjectTampered	193	ErrObjectTampered
		194
193	ErrHealNotImplemented	195	ErrHealNotImplemented
194	ErrHealNoSuchProcess	196	ErrHealNoSuchProcess
195	ErrHealInvalidClientToken	197	ErrHealInvalidClientToken
@@ -197,6 +199,93 @@ const ( · 1 unmodified line
197	ErrHealAlreadyRunning	199	ErrHealAlreadyRunning
198	ErrHealOverlappingPaths	200	ErrHealOverlappingPaths
199	ErrIncorrectContinuationToken	201	ErrIncorrectContinuationToken
		202
		203	//S3 Select Errors
		204	ErrEmptyRequestBody
		205	ErrUnsupportedFunction
		206	ErrInvalidExpressionType
		207	ErrBusy
		208	ErrUnauthorizedAccess
		209	ErrExpressionTooLong
		210	ErrIllegalSQLFunctionArgument
		211	ErrInvalidKeyPath
		212	ErrInvalidCompressionFormat
		213	ErrInvalidFileHeaderInfo
		214	ErrInvalidJSONType
		215	ErrInvalidQuoteFields
		216	ErrInvalidRequestParameter
		217	ErrInvalidDataType
		218	ErrInvalidTextEncoding
		219	ErrInvalidDataSource
		220	ErrInvalidTableAlias
		221	ErrMissingRequiredParameter
		222	ErrObjectSerializationConflict
		223	ErrUnsupportedSQLOperation
		224	ErrUnsupportedSQLStructure
		225	ErrUnsupportedSyntax
		226	ErrUnsupportedRangeHeader
		227	ErrLexerInvalidChar
		228	ErrLexerInvalidOperator
		229	ErrLexerInvalidLiteral
		230	ErrLexerInvalidIONLiteral
		231	ErrParseExpectedDatePart
		232	ErrParseExpectedKeyword
		233	ErrParseExpectedTokenType
		234	ErrParseExpected2TokenTypes
		235	ErrParseExpectedNumber
		236	ErrParseExpectedRightParenBuiltinFunctionCall
		237	ErrParseExpectedTypeName
		238	ErrParseExpectedWhenClause
		239	ErrParseUnsupportedToken
		240	ErrParseUnsupportedLiteralsGroupBy
		241	ErrParseExpectedMember
		242	ErrParseUnsupportedSelect
		243	ErrParseUnsupportedCase
		244	ErrParseUnsupportedCaseClause
		245	ErrParseUnsupportedAlias
		246	ErrParseUnsupportedSyntax
		247	ErrParseUnknownOperator
		248	ErrParseInvalidPathComponent
		249	ErrParseMissingIdentAfterAt
		250	ErrParseUnexpectedOperator
		251	ErrParseUnexpectedTerm
		252	ErrParseUnexpectedToken
		253	ErrParseUnexpectedKeyword
		254	ErrParseExpectedExpression
		255	ErrParseExpectedLeftParenAfterCast
		256	ErrParseExpectedLeftParenValueConstructor
		257	ErrParseExpectedLeftParenBuiltinFunctionCall
		258	ErrParseExpectedArgumentDelimiter
		259	ErrParseCastArity
		260	ErrParseInvalidTypeParam
		261	ErrParseEmptySelect
		262	ErrParseSelectMissingFrom
		263	ErrParseExpectedIdentForGroupName
		264	ErrParseExpectedIdentForAlias
		265	ErrParseUnsupportedCallWithStar
		266	ErrParseNonUnaryAgregateFunctionCall
		267	ErrParseMalformedJoin
		268	ErrParseExpectedIdentForAt
		269	ErrParseAsteriskIsNotAloneInSelectList
		270	ErrParseCannotMixSqbAndWildcardInSelectList
		271	ErrParseInvalidContextForWildcardInSelectList
		272	ErrIncorrectSQLFunctionArgumentType
		273	ErrValueParseFailure
		274	ErrEvaluatorInvalidArguments
		275	ErrIntegerOverflow
		276	ErrLikeInvalidInputs
		277	ErrCastFailed
		278	ErrInvalidCast
		279	ErrEvaluatorInvalidTimestampFormatPattern
		280	ErrEvaluatorInvalidTimestampFormatPatternSymbolForParsing
		281	ErrEvaluatorTimestampFormatPatternDuplicateFields
		282	ErrEvaluatorTimestampFormatPatternHourClockAmPmMismatch
		283	ErrEvaluatorUnterminatedTimestampFormatPatternToken
		284	ErrEvaluatorInvalidTimestampFormatPatternToken
		285	ErrEvaluatorInvalidTimestampFormatPatternSymbol
		286	ErrEvaluatorBindingDoesNotExist
		287	ErrInvalidColumnIndex
		288	ErrMissingHeaders
200	)	289	)
201		290
202	// error code to APIError structure, these fields carry respective	291	// error code to APIError structure, these fields carry respective
@@ -803,6 +892,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{ · 600 unmodified lines
803	Description: "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds",	892	Description: "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds",
804	HTTPStatusCode: http.StatusBadRequest,	893	HTTPStatusCode: http.StatusBadRequest,
805	},	894	},
		895
806	// Generic Invalid-Request error. Should be used for response errors only for unlikely	896	// Generic Invalid-Request error. Should be used for response errors only for unlikely
807	// corner case errors for which introducing new APIErrorCode is not worth it. LogIf()	897	// corner case errors for which introducing new APIErrorCode is not worth it. LogIf()
808	// should be used to log the error at the source of the error for debugging purposes.	898	// should be used to log the error at the source of the error for debugging purposes.
@@ -851,6 +941,432 @@ var errorCodeResponse = map[APIErrorCode]APIError{ · 42 unmodified lines
851	Description: "The continuation token provided is incorrect",	941	Description: "The continuation token provided is incorrect",
852	HTTPStatusCode: http.StatusBadRequest,	942	HTTPStatusCode: http.StatusBadRequest,
853	},	943	},
		944	//S3 Select API Errors
		945	ErrEmptyRequestBody: {
		946	Code: "EmptyRequestBody",
		947	Description: "Request body cannot be empty.",
		948	HTTPStatusCode: http.StatusBadRequest,
		949	},
		950	ErrUnsupportedFunction: {
		951	Code: "UnsupportedFunction",
		952	Description: "Encountered an unsupported SQL function.",
		953	HTTPStatusCode: http.StatusBadRequest,
		954	},
		955	ErrInvalidDataSource: {
		956	Code: "InvalidDataSource",
		957	Description: "Invalid data source type. Only CSV and JSON are supported at this time.",
		958	HTTPStatusCode: http.StatusBadRequest,
		959	},
		960	ErrInvalidExpressionType: {
		961	Code: "InvalidExpressionType",
		962	Description: "The ExpressionType is invalid. Only SQL expressions are supported at this time.",
		963	HTTPStatusCode: http.StatusBadRequest,
		964	},
		965	ErrBusy: {
		966	Code: "Busy",
		967	Description: "The service is unavailable. Please retry.",
		968	HTTPStatusCode: http.StatusServiceUnavailable,
		969	},
		970	ErrUnauthorizedAccess: {
		971	Code: "UnauthorizedAccess",
		972	Description: "You are not authorized to perform this operation",
		973	HTTPStatusCode: http.StatusUnauthorized,
		974	},
		975	ErrExpressionTooLong: {
		976	Code: "ExpressionTooLong",
		977	Description: "The SQL expression is too long: The maximum byte-length for the SQL expression is 256 KB.",
		978	HTTPStatusCode: http.StatusBadRequest,
		979	},
		980	ErrIllegalSQLFunctionArgument: {
		981	Code: "IllegalSqlFunctionArgument",
		982	Description: "Illegal argument was used in the SQL function.",
		983	HTTPStatusCode: http.StatusBadRequest,
		984	},
		985	ErrInvalidKeyPath: {
		986	Code: "InvalidKeyPath",
		987	Description: "Key path in the SQL expression is invalid.",
		988	HTTPStatusCode: http.StatusBadRequest,
		989	},
		990	ErrInvalidCompressionFormat: {
		991	Code: "InvalidCompressionFormat",
		992	Description: "The file is not in a supported compression format. Only GZIP is supported at this time.",
		993	HTTPStatusCode: http.StatusBadRequest,
		994	},
		995	ErrInvalidFileHeaderInfo: {
		996	Code: "InvalidFileHeaderInfo",
		997	Description: "The FileHeaderInfo is invalid. Only NONE, USE, and IGNORE are supported.",
		998	HTTPStatusCode: http.StatusBadRequest,
		999	},
		1000	ErrInvalidJSONType: {
		1001	Code: "InvalidJsonType",
		1002	Description: "The JsonType is invalid. Only DOCUMENT and LINES are supported at this time.",
		1003	HTTPStatusCode: http.StatusBadRequest,
		1004	},
		1005	ErrInvalidQuoteFields: {
		1006	Code: "InvalidQuoteFields",
		1007	Description: "The QuoteFields is invalid. Only ALWAYS and ASNEEDED are supported.",
		1008	HTTPStatusCode: http.StatusBadRequest,
		1009	},
		1010	ErrInvalidRequestParameter: {
		1011	Code: "InvalidRequestParameter",
		1012	Description: "The value of a parameter in SelectRequest element is invalid. Check the service API documentation and try again.",
		1013	HTTPStatusCode: http.StatusBadRequest,
		1014	},
		1015	ErrInvalidDataType: {
		1016	Code: "InvalidDataType",
		1017	Description: "The SQL expression contains an invalid data type.",
		1018	HTTPStatusCode: http.StatusBadRequest,
		1019	},
		1020	ErrInvalidTextEncoding: {
		1021	Code: "InvalidTextEncoding",
		1022	Description: "Invalid encoding type. Only UTF-8 encoding is supported at this time.",
		1023	HTTPStatusCode: http.StatusBadRequest,
		1024	},
		1025	ErrInvalidTableAlias: {
		1026	Code: "InvalidTableAlias",
		1027	Description: "The SQL expression contains an invalid table alias.",
		1028	HTTPStatusCode: http.StatusBadRequest,
		1029	},
		1030	ErrMissingRequiredParameter: {
		1031	Code: "MissingRequiredParameter",
		1032	Description: "The SelectRequest entity is missing a required parameter. Check the service documentation and try again.",
		1033	HTTPStatusCode: http.StatusBadRequest,
		1034	},
		1035	ErrObjectSerializationConflict: {
		1036	Code: "ObjectSerializationConflict",
		1037	Description: "The SelectRequest entity can only contain one of CSV or JSON. Check the service documentation and try again.",
		1038	HTTPStatusCode: http.StatusBadRequest,
		1039	},
		1040	ErrUnsupportedSQLOperation: {
		1041	Code: "UnsupportedSqlOperation",
		1042	Description: "Encountered an unsupported SQL operation.",
		1043	HTTPStatusCode: http.StatusBadRequest,
		1044	},
		1045	ErrUnsupportedSQLStructure: {
		1046	Code: "UnsupportedSqlStructure",
		1047	Description: "Encountered an unsupported SQL structure. Check the SQL Reference.",
		1048	HTTPStatusCode: http.StatusBadRequest,
		1049	},
		1050	ErrUnsupportedSyntax: {
		1051	Code: "UnsupportedSyntax",
		1052	Description: "Encountered invalid syntax.",
		1053	HTTPStatusCode: http.StatusBadRequest,
		1054	},
		1055	ErrUnsupportedRangeHeader: {
		1056	Code: "UnsupportedRangeHeader",
		1057	Description: "Range header is not supported for this operation.",
		1058	HTTPStatusCode: http.StatusBadRequest,
		1059	},
		1060	ErrLexerInvalidChar: {
		1061	Code: "LexerInvalidChar",
		1062	Description: "The SQL expression contains an invalid character.",
		1063	HTTPStatusCode: http.StatusBadRequest,
		1064	},
		1065	ErrLexerInvalidOperator: {
		1066	Code: "LexerInvalidOperator",
		1067	Description: "The SQL expression contains an invalid literal.",
		1068	HTTPStatusCode: http.StatusBadRequest,
		1069	},
		1070	ErrLexerInvalidLiteral: {
		1071	Code: "LexerInvalidLiteral",
		1072	Description: "The SQL expression contains an invalid operator.",
		1073	HTTPStatusCode: http.StatusBadRequest,
		1074	},
		1075	ErrLexerInvalidIONLiteral: {
		1076	Code: "LexerInvalidIONLiteral",
		1077	Description: "The SQL expression contains an invalid operator.",
		1078	HTTPStatusCode: http.StatusBadRequest,
		1079	},
		1080	ErrParseExpectedDatePart: {
		1081	Code: "ParseExpectedDatePart",
		1082	Description: "Did not find the expected date part in the SQL expression.",
		1083	HTTPStatusCode: http.StatusBadRequest,
		1084	},
		1085	ErrParseExpectedKeyword: {
		1086	Code: "ParseExpectedKeyword",
		1087	Description: "Did not find the expected keyword in the SQL expression.",
		1088	HTTPStatusCode: http.StatusBadRequest,
		1089	},
		1090	ErrParseExpectedTokenType: {
		1091	Code: "ParseExpectedTokenType",
		1092	Description: "Did not find the expected token in the SQL expression.",
		1093	HTTPStatusCode: http.StatusBadRequest,
		1094	},
		1095	ErrParseExpected2TokenTypes: {
		1096	Code: "ParseExpected2TokenTypes",
		1097	Description: "Did not find the expected token in the SQL expression.",
		1098	HTTPStatusCode: http.StatusBadRequest,
		1099	},
		1100	ErrParseExpectedNumber: {
		1101	Code: "ParseExpectedNumber",
		1102	Description: "Did not find the expected number in the SQL expression.",
		1103	HTTPStatusCode: http.StatusBadRequest,
		1104	},
		1105	ErrParseExpectedRightParenBuiltinFunctionCall: {
		1106	Code: "ParseExpectedRightParenBuiltinFunctionCall",
		1107	Description: "Did not find the expected right parenthesis character in the SQL expression.",
		1108	HTTPStatusCode: http.StatusBadRequest,
		1109	},
		1110	ErrParseExpectedTypeName: {
		1111	Code: "ParseExpectedTypeName",
		1112	Description: "Did not find the expected type name in the SQL expression.",
		1113	HTTPStatusCode: http.StatusBadRequest,
		1114	},
		1115	ErrParseExpectedWhenClause: {
		1116	Code: "ParseExpectedWhenClause",
		1117	Description: "Did not find the expected WHEN clause in the SQL expression. CASE is not supported.",
		1118	HTTPStatusCode: http.StatusBadRequest,
		1119	},
		1120	ErrParseUnsupportedToken: {
		1121	Code: "ParseUnsupportedToken",
		1122	Description: "The SQL expression contains an unsupported token.",
		1123	HTTPStatusCode: http.StatusBadRequest,
		1124	},
		1125	ErrParseUnsupportedLiteralsGroupBy: {
		1126	Code: "ParseUnsupportedLiteralsGroupBy",
		1127	Description: "The SQL expression contains an unsupported use of GROUP BY.",
		1128	HTTPStatusCode: http.StatusBadRequest,
		1129	},
		1130	ErrParseExpectedMember: {
		1131	Code: "ParseExpectedMember",
		1132	Description: "The SQL expression contains an unsupported use of MEMBER.",
		1133	HTTPStatusCode: http.StatusBadRequest,
		1134	},
		1135	ErrParseUnsupportedSelect: {
		1136	Code: "ParseUnsupportedSelect",
		1137	Description: "The SQL expression contains an unsupported use of SELECT.",
		1138	HTTPStatusCode: http.StatusBadRequest,
		1139	},
		1140	ErrParseUnsupportedCase: {
		1141	Code: "ParseUnsupportedCase",
		1142	Description: "The SQL expression contains an unsupported use of CASE.",
		1143	HTTPStatusCode: http.StatusBadRequest,
		1144	},
		1145	ErrParseUnsupportedCaseClause: {
		1146	Code: "ParseUnsupportedCaseClause",
		1147	Description: "The SQL expression contains an unsupported use of CASE.",
		1148	HTTPStatusCode: http.StatusBadRequest,
		1149	},
		1150	ErrParseUnsupportedAlias: {
		1151	Code: "ParseUnsupportedAlias",
		1152	Description: "The SQL expression contains an unsupported use of ALIAS.",
		1153	HTTPStatusCode: http.StatusBadRequest,
		1154	},
		1155	ErrParseUnsupportedSyntax: {
		1156	Code: "ParseUnsupportedSyntax",
		1157	Description: "The SQL expression contains unsupported syntax.",
		1158	HTTPStatusCode: http.StatusBadRequest,
		1159	},
		1160	ErrParseUnknownOperator: {
		1161	Code: "ParseUnknownOperator",
		1162	Description: "The SQL expression contains an invalid operator.",
		1163	HTTPStatusCode: http.StatusBadRequest,
		1164	},
		1165	ErrParseInvalidPathComponent: {
		1166	Code: "ParseInvalidPathComponent",
		1167	Description: "The SQL expression contains an invalid path component.",
		1168	HTTPStatusCode: http.StatusBadRequest,
		1169	},
		1170	ErrParseMissingIdentAfterAt: {
		1171	Code: "ParseMissingIdentAfterAt",
		1172	Description: "Did not find the expected identifier after the @ symbol in the SQL expression.",
		1173	HTTPStatusCode: http.StatusBadRequest,
		1174	},
		1175	ErrParseUnexpectedOperator: {
		1176	Code: "ParseUnexpectedOperator",
		1177	Description: "The SQL expression contains an unexpected operator.",
		1178	HTTPStatusCode: http.StatusBadRequest,
		1179	},
		1180	ErrParseUnexpectedTerm: {
		1181	Code: "ParseUnexpectedTerm",
		1182	Description: "The SQL expression contains an unexpected term.",
		1183	HTTPStatusCode: http.StatusBadRequest,
		1184	},
		1185	ErrParseUnexpectedToken: {
		1186	Code: "ParseUnexpectedToken",
		1187	Description: "The SQL expression contains an unexpected token.",
		1188	HTTPStatusCode: http.StatusBadRequest,
		1189	},
		1190	ErrParseUnexpectedKeyword: {
		1191	Code: "ParseUnexpectedKeyword",
		1192	Description: "The SQL expression contains an unexpected keyword.",
		1193	HTTPStatusCode: http.StatusBadRequest,
		1194	},
		1195	ErrParseExpectedExpression: {
		1196	Code: "ParseExpectedExpression",
		1197	Description: "Did not find the expected SQL expression.",
		1198	HTTPStatusCode: http.StatusBadRequest,
		1199	},
		1200	ErrParseExpectedLeftParenAfterCast: {
		1201	Code: "ParseExpectedLeftParenAfterCast",
		1202	Description: "Did not find expected the left parenthesis in the SQL expression.",
		1203	HTTPStatusCode: http.StatusBadRequest,
		1204	},
		1205	ErrParseExpectedLeftParenValueConstructor: {
		1206	Code: "ParseExpectedLeftParenValueConstructor",
		1207	Description: "Did not find expected the left parenthesis in the SQL expression.",
		1208	HTTPStatusCode: http.StatusBadRequest,
		1209	},
		1210	ErrParseExpectedLeftParenBuiltinFunctionCall: {
		1211	Code: "ParseExpectedLeftParenBuiltinFunctionCall",
		1212	Description: "Did not find the expected left parenthesis in the SQL expression.",
		1213	HTTPStatusCode: http.StatusBadRequest,
		1214	},
		1215	ErrParseExpectedArgumentDelimiter: {
		1216	Code: "ParseExpectedArgumentDelimiter",
		1217	Description: "Did not find the expected argument delimiter in the SQL expression.",
		1218	HTTPStatusCode: http.StatusBadRequest,
		1219	},
		1220	ErrParseCastArity: {
		1221	Code: "ParseCastArity",
		1222	Description: "The SQL expression CAST has incorrect arity.",
		1223	HTTPStatusCode: http.StatusBadRequest,
		1224	},
		1225	ErrParseInvalidTypeParam: {
		1226	Code: "ParseInvalidTypeParam",
		1227	Description: "The SQL expression contains an invalid parameter value.",
		1228	HTTPStatusCode: http.StatusBadRequest,
		1229	},
		1230	ErrParseEmptySelect: {
		1231	Code: "ParseEmptySelect",
		1232	Description: "The SQL expression contains an empty SELECT.",
		1233	HTTPStatusCode: http.StatusBadRequest,
		1234	},
		1235	ErrParseSelectMissingFrom: {
		1236	Code: "ParseSelectMissingFrom",
		1237	Description: "GROUP is not supported in the SQL expression.",
		1238	HTTPStatusCode: http.StatusBadRequest,
		1239	},
		1240	ErrParseExpectedIdentForGroupName: {
		1241	Code: "ParseExpectedIdentForGroupName",
		1242	Description: "GROUP is not supported in the SQL expression.",
		1243	HTTPStatusCode: http.StatusBadRequest,
		1244	},
		1245	ErrParseExpectedIdentForAlias: {
		1246	Code: "ParseExpectedIdentForAlias",
		1247	Description: "Did not find the expected identifier for the alias in the SQL expression.",
		1248	HTTPStatusCode: http.StatusBadRequest,
		1249	},
		1250	ErrParseUnsupportedCallWithStar: {
		1251	Code: "ParseUnsupportedCallWithStar",
		1252	Description: "Only COUNT with (*) as a parameter is supported in the SQL expression.",
		1253	HTTPStatusCode: http.StatusBadRequest,
		1254	},
		1255	ErrParseNonUnaryAgregateFunctionCall: {
		1256	Code: "ParseNonUnaryAgregateFunctionCall",
		1257	Description: "Only one argument is supported for aggregate functions in the SQL expression.",
		1258	HTTPStatusCode: http.StatusBadRequest,
		1259	},
		1260	ErrParseMalformedJoin: {
		1261	Code: "ParseMalformedJoin",
		1262	Description: "JOIN is not supported in the SQL expression.",
		1263	HTTPStatusCode: http.StatusBadRequest,
		1264	},
		1265	ErrParseExpectedIdentForAt: {
		1266	Code: "ParseExpectedIdentForAt",
		1267	Description: "Did not find the expected identifier for AT name in the SQL expression.",
		1268	HTTPStatusCode: http.StatusBadRequest,
		1269	},
		1270	ErrParseAsteriskIsNotAloneInSelectList: {
		1271	Code: "ParseAsteriskIsNotAloneInSelectList",
		1272	Description: "Other expressions are not allowed in the SELECT list when '*' is used without dot notation in the SQL expression.",
		1273	HTTPStatusCode: http.StatusBadRequest,
		1274	},
		1275	ErrParseCannotMixSqbAndWildcardInSelectList: {
		1276	Code: "ParseCannotMixSqbAndWildcardInSelectList",
		1277	Description: "Cannot mix [] and * in the same expression in a SELECT list in SQL expression.",
		1278	HTTPStatusCode: http.StatusBadRequest,
		1279	},
		1280	ErrParseInvalidContextForWildcardInSelectList: {
		1281	Code: "ParseInvalidContextForWildcardInSelectList",
		1282	Description: "Invalid use of * in SELECT list in the SQL expression.",
		1283	HTTPStatusCode: http.StatusBadRequest,
		1284	},
		1285	ErrIncorrectSQLFunctionArgumentType: {
		1286	Code: "IncorrectSqlFunctionArgumentType",
		1287	Description: "Incorrect type of arguments in function call in the SQL expression.",
		1288	HTTPStatusCode: http.StatusBadRequest,
		1289	},
		1290	ErrValueParseFailure: {
		1291	Code: "ValueParseFailure",
		1292	Description: "Time stamp parse failure in the SQL expression.",
		1293	HTTPStatusCode: http.StatusBadRequest,
		1294	},
		1295	ErrEvaluatorInvalidArguments: {
		1296	Code: "EvaluatorInvalidArguments",
		1297	Description: "Incorrect number of arguments in the function call in the SQL expression.",
		1298	HTTPStatusCode: http.StatusBadRequest,
		1299	},
		1300	ErrIntegerOverflow: {
		1301	Code: "IntegerOverflow",
		1302	Description: "Int overflow or underflow in the SQL expression.",
		1303	HTTPStatusCode: http.StatusBadRequest,
		1304	},
		1305	ErrLikeInvalidInputs: {
		1306	Code: "LikeInvalidInputs",
		1307	Description: "Invalid argument given to the LIKE clause in the SQL expression.",
		1308	HTTPStatusCode: http.StatusBadRequest,
		1309	},
		1310	ErrCastFailed: {
		1311	Code: "CastFailed",
		1312	Description: "Attempt to convert from one data type to another using CAST failed in the SQL expression.",
		1313	HTTPStatusCode: http.StatusBadRequest,
		1314	},
		1315	ErrInvalidCast: {
		1316	Code: "InvalidCast",
		1317	Description: "Attempt to convert from one data type to another using CAST failed in the SQL expression.",
		1318	HTTPStatusCode: http.StatusBadRequest,
		1319	},
		1320	ErrEvaluatorInvalidTimestampFormatPattern: {
		1321	Code: "EvaluatorInvalidTimestampFormatPattern",
		1322	Description: "Time stamp format pattern requires additional fields in the SQL expression.",
		1323	HTTPStatusCode: http.StatusBadRequest,
		1324	},
		1325	ErrEvaluatorInvalidTimestampFormatPatternSymbolForParsing: {
		1326	Code: "EvaluatorInvalidTimestampFormatPatternSymbolForParsing",
		1327	Description: "Time stamp format pattern contains a valid format symbol that cannot be applied to time stamp parsing in the SQL expression.",
		1328	HTTPStatusCode: http.StatusBadRequest,
		1329	},
		1330	ErrEvaluatorTimestampFormatPatternDuplicateFields: {
		1331	Code: "EvaluatorTimestampFormatPatternDuplicateFields",
		1332	Description: "Time stamp format pattern contains multiple format specifiers representing the time stamp field in the SQL expression.",
		1333	HTTPStatusCode: http.StatusBadRequest,
		1334	},
		1335	ErrEvaluatorTimestampFormatPatternHourClockAmPmMismatch: {
		1336	Code: "EvaluatorUnterminatedTimestampFormatPatternToken",
		1337	Description: "Time stamp format pattern contains unterminated token in the SQL expression.",
		1338	HTTPStatusCode: http.StatusBadRequest,
		1339	},
		1340	ErrEvaluatorUnterminatedTimestampFormatPatternToken: {
		1341	Code: "EvaluatorInvalidTimestampFormatPatternToken",
		1342	Description: "Time stamp format pattern contains an invalid token in the SQL expression.",
		1343	HTTPStatusCode: http.StatusBadRequest,
		1344	},
		1345	ErrEvaluatorInvalidTimestampFormatPatternToken: {
		1346	Code: "EvaluatorInvalidTimestampFormatPatternToken",
		1347	Description: "Time stamp format pattern contains an invalid token in the SQL expression.",
		1348	HTTPStatusCode: http.StatusBadRequest,
		1349	},
		1350	ErrEvaluatorInvalidTimestampFormatPatternSymbol: {
		1351	Code: "EvaluatorInvalidTimestampFormatPatternSymbol",
		1352	Description: "Time stamp format pattern contains an invalid symbol in the SQL expression.",
		1353	HTTPStatusCode: http.StatusBadRequest,
		1354	},
		1355	ErrInvalidColumnIndex: {
		1356	Code: "InvalidColumnIndex",
		1357	Description: "Column index in the SQL expression is invalid.",
		1358	HTTPStatusCode: http.StatusBadRequest,
		1359	},
		1360	ErrEvaluatorBindingDoesNotExist: {
		1361	Code: "ErrEvaluatorBindingDoesNotExist",
		1362	Description: "A column name or a path provided does not exist in the SQL expression",
		1363	HTTPStatusCode: http.StatusBadRequest,
		1364	},
		1365	ErrMissingHeaders: {
		1366	Code: "MissingHeaders",
		1367	Description: "Some headers in the query are missing from the file. Check the file and try again.",
		1368	HTTPStatusCode: http.StatusBadRequest,
		1369	},
854	// Add your error structure here.	1370	// Add your error structure here.
855	}	1371	}
856		1372
@@ -900,6 +1416,169 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) { · 43 unmodified lines
900	case context.Canceled, context.DeadlineExceeded:	1416	case context.Canceled, context.DeadlineExceeded:
901	apiErr = ErrOperationTimedOut	1417	apiErr = ErrOperationTimedOut
902	}	1418	}
		1419	switch err {
		1420	case s3select.ErrBusy:
		1421	apiErr = ErrBusy
		1422	case s3select.ErrUnauthorizedAccess:
		1423	apiErr = ErrUnauthorizedAccess
		1424	case s3select.ErrExpressionTooLong:
		1425	apiErr = ErrExpressionTooLong
		1426	case s3select.ErrIllegalSQLFunctionArgument:
		1427	apiErr = ErrIllegalSQLFunctionArgument
		1428	case s3select.ErrInvalidKeyPath:
		1429	apiErr = ErrInvalidKeyPath
		1430	case s3select.ErrInvalidCompressionFormat:
		1431	apiErr = ErrInvalidCompressionFormat
		1432	case s3select.ErrInvalidFileHeaderInfo:
		1433	apiErr = ErrInvalidFileHeaderInfo
		1434	case s3select.ErrInvalidJSONType:
		1435	apiErr = ErrInvalidJSONType
		1436	case s3select.ErrInvalidQuoteFields:
		1437	apiErr = ErrInvalidQuoteFields
		1438	case s3select.ErrInvalidRequestParameter:
		1439	apiErr = ErrInvalidRequestParameter
		1440	case s3select.ErrInvalidDataType:
		1441	apiErr = ErrInvalidDataType
		1442	case s3select.ErrInvalidTextEncoding:
		1443	apiErr = ErrInvalidTextEncoding
		1444	case s3select.ErrInvalidTableAlias:
		1445	apiErr = ErrInvalidTableAlias
		1446	case s3select.ErrMissingRequiredParameter:
		1447	apiErr = ErrMissingRequiredParameter
		1448	case s3select.ErrObjectSerializationConflict:
		1449	apiErr = ErrObjectSerializationConflict
		1450	case s3select.ErrUnsupportedSQLOperation:
		1451	apiErr = ErrUnsupportedSQLOperation
		1452	case s3select.ErrUnsupportedSQLStructure:
		1453	apiErr = ErrUnsupportedSQLStructure
		1454	case s3select.ErrUnsupportedSyntax:
		1455	apiErr = ErrUnsupportedSyntax
		1456	case s3select.ErrUnsupportedRangeHeader:
		1457	apiErr = ErrUnsupportedRangeHeader
		1458	case s3select.ErrLexerInvalidChar:
		1459	apiErr = ErrLexerInvalidChar
		1460	case s3select.ErrLexerInvalidOperator:
		1461	apiErr = ErrLexerInvalidOperator
		1462	case s3select.ErrLexerInvalidLiteral:
		1463	apiErr = ErrLexerInvalidLiteral
		1464	case s3select.ErrLexerInvalidIONLiteral:
		1465	apiErr = ErrLexerInvalidIONLiteral
		1466	case s3select.ErrParseExpectedDatePart:
		1467	apiErr = ErrParseExpectedDatePart
		1468	case s3select.ErrParseExpectedKeyword:
		1469	apiErr = ErrParseExpectedKeyword
		1470	case s3select.ErrParseExpectedTokenType:
		1471	apiErr = ErrParseExpectedTokenType
		1472	case s3select.ErrParseExpected2TokenTypes:
		1473	apiErr = ErrParseExpected2TokenTypes
		1474	case s3select.ErrParseExpectedNumber:
		1475	apiErr = ErrParseExpectedNumber
		1476	case s3select.ErrParseExpectedRightParenBuiltinFunctionCall:
		1477	apiErr = ErrParseExpectedRightParenBuiltinFunctionCall
		1478	case s3select.ErrParseExpectedTypeName:
		1479	apiErr = ErrParseExpectedTypeName
		1480	case s3select.ErrParseExpectedWhenClause:
		1481	apiErr = ErrParseExpectedWhenClause
		1482	case s3select.ErrParseUnsupportedToken:
		1483	apiErr = ErrParseUnsupportedToken
		1484	case s3select.ErrParseUnsupportedLiteralsGroupBy:
		1485	apiErr = ErrParseUnsupportedLiteralsGroupBy
		1486	case s3select.ErrParseExpectedMember:
		1487	apiErr = ErrParseExpectedMember
		1488	case s3select.ErrParseUnsupportedSelect:
		1489	apiErr = ErrParseUnsupportedSelect
		1490	case s3select.ErrParseUnsupportedCase:
		1491	apiErr = ErrParseUnsupportedCase
		1492	case s3select.ErrParseUnsupportedCaseClause:
		1493	apiErr = ErrParseUnsupportedCaseClause
		1494	case s3select.ErrParseUnsupportedAlias:
		1495	apiErr = ErrParseUnsupportedAlias
		1496	case s3select.ErrParseUnsupportedSyntax:
		1497	apiErr = ErrParseUnsupportedSyntax
		1498	case s3select.ErrParseUnknownOperator:
		1499	apiErr = ErrParseUnknownOperator
		1500	case s3select.ErrParseInvalidPathComponent:
		1501	apiErr = ErrParseInvalidPathComponent
		1502	case s3select.ErrParseMissingIdentAfterAt:
		1503	apiErr = ErrParseMissingIdentAfterAt
		1504	case s3select.ErrParseUnexpectedOperator:
		1505	apiErr = ErrParseUnexpectedOperator
		1506	case s3select.ErrParseUnexpectedTerm:
		1507	apiErr = ErrParseUnexpectedTerm
		1508	case s3select.ErrParseUnexpectedToken:
		1509	apiErr = ErrParseUnexpectedToken
		1510	case s3select.ErrParseUnexpectedKeyword:
		1511	apiErr = ErrParseUnexpectedKeyword
		1512	case s3select.ErrParseExpectedExpression:
		1513	apiErr = ErrParseExpectedExpression
		1514	case s3select.ErrParseExpectedLeftParenAfterCast:
		1515	apiErr = ErrParseExpectedLeftParenAfterCast
		1516	case s3select.ErrParseExpectedLeftParenValueConstructor:
		1517	apiErr = ErrParseExpectedLeftParenValueConstructor
		1518	case s3select.ErrParseExpectedLeftParenBuiltinFunctionCall:
		1519	apiErr = ErrParseExpectedLeftParenBuiltinFunctionCall
		1520	case s3select.ErrParseExpectedArgumentDelimiter:
		1521	apiErr = ErrParseExpectedArgumentDelimiter
		1522	case s3select.ErrParseCastArity:
		1523	apiErr = ErrParseCastArity
		1524	case s3select.ErrParseInvalidTypeParam:
		1525	apiErr = ErrParseInvalidTypeParam
		1526	case s3select.ErrParseEmptySelect:
		1527	apiErr = ErrParseEmptySelect
		1528	case s3select.ErrParseSelectMissingFrom:
		1529	apiErr = ErrParseSelectMissingFrom
		1530	case s3select.ErrParseExpectedIdentForGroupName:
		1531	apiErr = ErrParseExpectedIdentForGroupName
		1532	case s3select.ErrParseExpectedIdentForAlias:
		1533	apiErr = ErrParseExpectedIdentForAlias
		1534	case s3select.ErrParseUnsupportedCallWithStar:
		1535	apiErr = ErrParseUnsupportedCallWithStar
		1536	case s3select.ErrParseNonUnaryAgregateFunctionCall:
		1537	apiErr = ErrParseNonUnaryAgregateFunctionCall
		1538	case s3select.ErrParseMalformedJoin:
		1539	apiErr = ErrParseMalformedJoin
		1540	case s3select.ErrParseExpectedIdentForAt:
		1541	apiErr = ErrParseExpectedIdentForAt
		1542	case s3select.ErrParseAsteriskIsNotAloneInSelectList:
		1543	apiErr = ErrParseAsteriskIsNotAloneInSelectList
		1544	case s3select.ErrParseCannotMixSqbAndWildcardInSelectList:
		1545	apiErr = ErrParseCannotMixSqbAndWildcardInSelectList
		1546	case s3select.ErrParseInvalidContextForWildcardInSelectList:
		1547	apiErr = ErrParseInvalidContextForWildcardInSelectList
		1548	case s3select.ErrIncorrectSQLFunctionArgumentType:
		1549	apiErr = ErrIncorrectSQLFunctionArgumentType
		1550	case s3select.ErrValueParseFailure:
		1551	apiErr = ErrValueParseFailure
		1552	case s3select.ErrIntegerOverflow:
		1553	apiErr = ErrIntegerOverflow
		1554	case s3select.ErrLikeInvalidInputs:
		1555	apiErr = ErrLikeInvalidInputs
		1556	case s3select.ErrCastFailed:
		1557	apiErr = ErrCastFailed
		1558	case s3select.ErrInvalidCast:
		1559	apiErr = ErrInvalidCast
		1560	case s3select.ErrEvaluatorInvalidTimestampFormatPattern:
		1561	apiErr = ErrEvaluatorInvalidTimestampFormatPattern
		1562	case s3select.ErrEvaluatorInvalidTimestampFormatPatternSymbolForParsing:
		1563	apiErr = ErrEvaluatorInvalidTimestampFormatPatternSymbolForParsing
		1564	case s3select.ErrEvaluatorTimestampFormatPatternDuplicateFields:
		1565	apiErr = ErrEvaluatorTimestampFormatPatternDuplicateFields
		1566	case s3select.ErrEvaluatorTimestampFormatPatternHourClockAmPmMismatch:
		1567	apiErr = ErrEvaluatorTimestampFormatPatternHourClockAmPmMismatch
		1568	case s3select.ErrEvaluatorUnterminatedTimestampFormatPatternToken:
		1569	apiErr = ErrEvaluatorUnterminatedTimestampFormatPatternToken
		1570	case s3select.ErrEvaluatorInvalidTimestampFormatPatternToken:
		1571	apiErr = ErrEvaluatorInvalidTimestampFormatPatternToken
		1572	case s3select.ErrEvaluatorInvalidTimestampFormatPatternSymbol:
		1573	apiErr = ErrEvaluatorInvalidTimestampFormatPatternSymbol
		1574	case s3select.ErrInvalidColumnIndex:
		1575	apiErr = ErrInvalidColumnIndex
		1576	case s3select.ErrEvaluatorBindingDoesNotExist:
		1577	apiErr = ErrEvaluatorBindingDoesNotExist
		1578	case s3select.ErrMissingHeaders:
		1579	apiErr = ErrMissingHeaders
		1580
		1581	}
903		1582
904	if apiErr != ErrNone {	1583	if apiErr != ErrNone {
905	// If there was a match in the above switch case.	1584	// If there was a match in the above switch case.

cmd/api-router.go+2−0

 		bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
 		// GetObjectACL - this is a dummy call.
 		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectACLHandler)).Queries("acl", "")
+		// SelectObjectContent
+		bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.SelectObjectContentHandler)).Queries("select", "").Queries("select-type", "2")
 		// GetObject
 		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectHandler))
 		// CopyObject

cmd/generic-handlers.go+0−1

@@ -475,7 +475,6 @@ var notimplementedObjectResourceNames = map[string]bool{ · 474 unmodified lines
475	"policy": true,	475	"policy": true,
476	"tagging": true,	476	"tagging": true,
477	"restore": true,	477	"restore": true,
478	"select": true,
479	}	478	}
480		479
481	// Resource handler ServeHTTP() wrapper	480	// Resource handler ServeHTTP() wrapper

cmd/object-handlers.go+187−0

@@ -30,6 +30,7 @@ import ( · 29 unmodified lines
30	"net/url"	30	"net/url"
31	"sort"	31	"sort"
32	"strconv"	32	"strconv"
		33	"strings"
33		34
34	"github.com/gorilla/mux"	35	"github.com/gorilla/mux"
35	miniogo "github.com/minio/minio-go"	36	miniogo "github.com/minio/minio-go"
@@ -40,6 +41,7 @@ import ( · 4 unmodified lines
40	"github.com/minio/minio/pkg/hash"	41	"github.com/minio/minio/pkg/hash"
41	"github.com/minio/minio/pkg/ioutil"	42	"github.com/minio/minio/pkg/ioutil"
42	"github.com/minio/minio/pkg/policy"	43	"github.com/minio/minio/pkg/policy"
		44	"github.com/minio/minio/pkg/s3select"
43	sha256 "github.com/minio/sha256-simd"	45	sha256 "github.com/minio/sha256-simd"
44	"github.com/minio/sio"	46	"github.com/minio/sio"
45	)	47	)
@@ -63,6 +65,191 @@ func setHeadGetRespHeaders(w http.ResponseWriter, reqParams url.Values) { · 17 unmodified lines
63	}	65	}
64	}	66	}
65		67
		68	// SelectObjectContentHandler - GET Object?select
		69	// ----------
		70	// This implementation of the GET operation retrieves object content based
		71	// on an SQL expression. In the request, along with the sql expression, you must
		72	// also specify a data serialization format (JSON, CSV) of the object.
		73	func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r *http.Request) {
		74	ctx := newContext(r, w, "SelectObject")
		75	var object, bucket string
		76	vars := mux.Vars(r)
		77	bucket = vars["bucket"]
		78	object = vars["object"]
		79
		80	// Fetch object stat info.
		81	objectAPI := api.ObjectAPI()
		82	if objectAPI == nil {
		83	writeErrorResponse(w, ErrServerNotInitialized, r.URL)
		84	return
		85	}
		86
		87	getObjectInfo := objectAPI.GetObjectInfo
		88	if api.CacheAPI() != nil {
		89	getObjectInfo = api.CacheAPI().GetObjectInfo
		90	}
		91
		92	if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectAction, bucket, object); s3Error != ErrNone {
		93	if getRequestAuthType(r) == authTypeAnonymous {
		94	// As per "Permission" section in
		95	// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html If
		96	// the object you request does not exist, the error Amazon S3 returns
		97	// depends on whether you also have the s3:ListBucket permission. * If you
		98	// have the s3:ListBucket permission on the bucket, Amazon S3 will return
		99	// an HTTP status code 404 ("no such key") error. * if you don’t have the
		100	// s3:ListBucket permission, Amazon S3 will return an HTTP status code 403
		101	// ("access denied") error.`
		102	if globalPolicySys.IsAllowed(policy.Args{
		103	Action: policy.ListBucketAction,
		104	BucketName: bucket,
		105	ConditionValues: getConditionValues(r, ""),
		106	IsOwner: false,
		107	}) {
		108	_, err := getObjectInfo(ctx, bucket, object)
		109	if toAPIErrorCode(err) == ErrNoSuchKey {
		110	s3Error = ErrNoSuchKey
		111	}
		112	}
		113	}
		114	writeErrorResponse(w, s3Error, r.URL)
		115	return
		116	}
		117	if r.ContentLength <= 0 {
		118	writeErrorResponse(w, ErrEmptyRequestBody, r.URL)
		119	return
		120	}
		121	var selectReq ObjectSelectRequest
		122	if err := xmlDecoder(r.Body, &selectReq, r.ContentLength); err != nil {
		123	fmt.Println(err)
		124	writeErrorResponse(w, ErrMalformedXML, r.URL)
		125	return
		126	}
		127
		128	objInfo, err := getObjectInfo(ctx, bucket, object)
		129	if err != nil {
		130	writeErrorResponse(w, toAPIErrorCode(err), r.URL)
		131	return
		132	}
		133	// Get request range.
		134	rangeHeader := r.Header.Get("Range")
		135	if rangeHeader != "" {
		136	writeErrorResponse(w, ErrUnsupportedRangeHeader, r.URL)
		137	return
		138	}
		139
		140	if selectReq.InputSerialization.CompressionType == SelectCompressionGZIP {
		141	if !strings.Contains(objInfo.ContentType, "gzip") {
		142	writeErrorResponse(w, ErrInvalidDataSource, r.URL)
		143	return
		144	}
		145	}
		146	if selectReq.InputSerialization.CompressionType == SelectCompressionBZIP {
		147	if !strings.Contains(objInfo.ContentType, "bzip") {
		148	writeErrorResponse(w, ErrInvalidDataSource, r.URL)
		149	return
		150	}
		151	}
		152	if selectReq.InputSerialization.CompressionType == SelectCompressionNONE \|\|
		153	selectReq.InputSerialization.CompressionType == "" {
		154	selectReq.InputSerialization.CompressionType = SelectCompressionNONE
		155	if !strings.Contains(objInfo.ContentType, "text/csv") {
		156	writeErrorResponse(w, ErrInvalidDataSource, r.URL)
		157	return
		158	}
		159	}
		160	if !strings.EqualFold(string(selectReq.ExpressionType), "SQL") {
		161	writeErrorResponse(w, ErrInvalidExpressionType, r.URL)
		162	return
		163	}
		164	if len(selectReq.Expression) >= (256 * 1000) {
		165	writeErrorResponse(w, ErrExpressionTooLong, r.URL)
		166	}
		167	if selectReq.InputSerialization.CSV.FileHeaderInfo != CSVFileHeaderInfoUse &&
		168	selectReq.InputSerialization.CSV.FileHeaderInfo != CSVFileHeaderInfoNone &&
		169	selectReq.InputSerialization.CSV.FileHeaderInfo != CSVFileHeaderInfoIgnore &&
		170	selectReq.InputSerialization.CSV.FileHeaderInfo != "" {
		171	writeErrorResponse(w, ErrInvalidFileHeaderInfo, r.URL)
		172	}
		173	if selectReq.OutputSerialization.CSV.QuoteFields != CSVQuoteFieldsAlways &&
		174	selectReq.OutputSerialization.CSV.QuoteFields != CSVQuoteFieldsAsNeeded &&
		175	selectReq.OutputSerialization.CSV.QuoteFields != "" {
		176	writeErrorResponse(w, ErrInvalidQuoteFields, r.URL)
		177	}
		178
		179	getObject := objectAPI.GetObject
		180	if api.CacheAPI() != nil && !hasSSECustomerHeader(r.Header) {
		181	getObject = api.CacheAPI().GetObject
		182	}
		183
		184	reader, pipewriter := io.Pipe()
		185
		186	// Get the object.
		187	var startOffset int64
		188	length := objInfo.Size
		189
		190	var writer io.Writer
		191	writer = pipewriter
		192	if objectAPI.IsEncryptionSupported() {
		193	if hasSSECustomerHeader(r.Header) {
		194	// Response writer should be limited early on for decryption upto required length,
		195	// additionally also skipping mod(offset)64KiB boundaries.
		196	writer = ioutil.LimitedWriter(writer, startOffset%(64*1024), length)
		197
		198	writer, startOffset, length, err = DecryptBlocksRequest(writer, r, bucket,
		199	object, startOffset, length, objInfo, false)
		200	if err != nil {
		201	writeErrorResponse(w, toAPIErrorCode(err), r.URL)
		202	return
		203	}
		204	}
		205	}
		206	go func() {
		207	defer reader.Close()
		208	if gerr := getObject(ctx, bucket, object, 0, objInfo.Size, writer,
		209	objInfo.ETag); gerr != nil {
		210	pipewriter.CloseWithError(gerr)
		211	return
		212	}
		213	pipewriter.Close() // Close writer explicitly signaling we wrote all data.
		214	}()
		215
		216	//s3select //Options
		217	if selectReq.OutputSerialization.CSV.FieldDelimiter == "" {
		218	selectReq.OutputSerialization.CSV.FieldDelimiter = ","
		219	}
		220	if selectReq.InputSerialization.CSV.FileHeaderInfo == "" {
		221	selectReq.InputSerialization.CSV.FileHeaderInfo = CSVFileHeaderInfoNone
		222	}
		223	if selectReq.InputSerialization.CSV != nil {
		224	options := &s3select.Options{
		225	HasHeader: selectReq.InputSerialization.CSV.FileHeaderInfo != CSVFileHeaderInfoNone,
		226	FieldDelimiter: selectReq.InputSerialization.CSV.FieldDelimiter,
		227	Comments: selectReq.InputSerialization.CSV.Comments,
		228	Name: "S3Object", // Default table name for all objects
		229	ReadFrom: reader,
		230	Compressed: string(selectReq.InputSerialization.CompressionType),
		231	Expression: selectReq.Expression,
		232	OutputFieldDelimiter: selectReq.OutputSerialization.CSV.FieldDelimiter,
		233	StreamSize: objInfo.Size,
		234	HeaderOpt: selectReq.InputSerialization.CSV.FileHeaderInfo == CSVFileHeaderInfoUse,
		235	}
		236	s3s, err := s3select.NewInput(options)
		237	if err != nil {
		238	writeErrorResponse(w, toAPIErrorCode(err), r.URL)
		239	return
		240	}
		241	_, _, _, _, _, _, err = s3s.ParseSelect(selectReq.Expression)
		242	if err != nil {
		243	writeErrorResponse(w, toAPIErrorCode(err), r.URL)
		244	return
		245	}
		246	if err := s3s.Execute(w); err != nil {
		247	logger.LogIf(ctx, err)
		248	}
		249	return
		250	}
		251	}
		252
66	// GetObjectHandler - GET Object	253	// GetObjectHandler - GET Object
67	// ----------	254	// ----------
68	// This implementation of the GET operation retrieves object. To use GET,	255	// This implementation of the GET operation retrieves object. To use GET,

docs/select/README.md+98−0

@@ -0,0 +1,98 @@
	1	# How to use Minio S3 Select [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io)
	2
	3	This document explains current limitations of the Minio S3 Select support.
	4
	5	## 1. Features to be implemented
	6	1). JSON documents as supported Objects
	7
	8	2). CAST expression
	9
	10	3). Date Functions
	11
	12	4). Returning types other than float from aggregation queries.
	13
	14	5). Bracket and Reversal Notation with SQL Like operator.
	15
	16	6). SUBSTRING currently is not supported and TRIM only works with default arguments of trim leading and trailing spaces
	17
	18	## 2. Sample Usage with AWS Boto Client
	19	```python
	20	import boto3
	21	from botocore.client import Config
	22	import os
	23	s3 = boto3.resource('s3',
	24	endpoint_url='ENDPOINT',
	25	aws_access_key_id='ACCESSKEY',
	26	aws_secret_access_key='SECRETKEY',
	27	config=Config(signature_version='s3v4'),
	28	region_name='us-east-1')
	29	s3_client = s3.meta.client
	30
	31	r = s3_client.select_object_content(
	32	Bucket='myBucket',
	33	Key='myKey',
	34	ExpressionType='SQL',
	35	Expression = "SELECT * FROM S3OBJECT AS A",
	36	InputSerialization = {'CSV': {"FieldDelimiter": ",","FileHeaderInfo":"USE"}},
	37	OutputSerialization = {'CSV': {}},
	38	)
	39	```
	40	## 3. Sample Usage with Minio-Go Client
	41
	42	```go
	43	// Initialize minio client object.
	44	minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, useSSL)
	45	if err != nil {
	46	log.Fatalln(err)
	47	}
	48	input := minio.SelectObjectInput{
	49	RecordDelimiter: "\n",
	50	FieldDelimiter: ",",
	51	FileHeaderInfo: minio.CSVFileHeaderInfoUse,
	52	}
	53	output := minio.SelectObjectOutput{
	54	RecordDelimiter: "\n",
	55	FieldDelimiter: ",",
	56	}
	57	opts := minio.SelectObjectOptions{
	58	Type: minio.SelectObjectTypeCSV,
	59	Input: input,
	60	Output: output,
	61	}
	62	myReader, err := minioClient.SelectObjectContent(ctx, "sqlselectapi", "player.csv", "Select * from S3OBJECT WHERE last_name = 'James'", opts)
	63	if err != nil {
	64	fmt.Println(err)
	65	return
	66	}
	67	defer myReader.Close()
	68
	69	results, resultWriter := io.Pipe()
	70	go func() {
	71	defer resultWriter.Close()
	72	for event := range myReader.Events() {
	73	switch e := event.(type) {
	74	case *minio.RecordEvent:
	75	resultWriter.Write(e.Payload)
	76	case *minio.ProgressEvent:
	77	fmt.Println("Progress")
	78	case *minio.StatEvent:
	79	fmt.Println(string(e.Payload))
	80	case *minio.EndEvent:
	81	fmt.Println("Ended")
	82	return
	83	}
	84	}
	85	}()
	86	resReader := csv.NewReader(results)
	87	for {
	88	record, err := resReader.Read()
	89	if err == io.EOF {
	90	break
	91	}
	92	// Print out the records
	93	fmt.Println(record)
	94	}
	95	if err := myReader.Err(); err != nil {
	96	fmt.Println(err)
	97	}
	98	```

pkg/s3select/errors.go+486−0

@@ -0,0 +1,486 @@
	1	/*
	2	* Minio Cloud Storage, (C) 2018 Minio, Inc.
	3	*
	4	* Licensed under the Apache License, Version 2.0 (the "License");
	5	* you may not use this file except in compliance with the License.
	6	* You may obtain a copy of the License at
	7	*
	8	* http://www.apache.org/licenses/LICENSE-2.0
	9	*
	10	* Unless required by applicable law or agreed to in writing, software
	11	* distributed under the License is distributed on an "AS IS" BASIS,
	12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	* See the License for the specific language governing permissions and
	14	* limitations under the License.
	15	*/
	16
	17	package s3select
	18
	19	import "errors"
	20
	21	//S3 errors below
	22
	23	// ErrBusy is an error if the service is too busy.
	24	var ErrBusy = errors.New("The service is unavailable. Please retry")
	25
	26	// ErrUnauthorizedAccess is an error if you lack the appropriate credentials to
	27	// access the object.
	28	var ErrUnauthorizedAccess = errors.New("You are not authorized to perform this operation")
	29
	30	// ErrExpressionTooLong is an error if your SQL expression too long for
	31	// processing.
	32	var ErrExpressionTooLong = errors.New("The SQL expression is too long: The maximum byte-length for the SQL expression is 256 KB")
	33
	34	// ErrIllegalSQLFunctionArgument is an error if you provide an illegal argument
	35	// in the SQL function.
	36	var ErrIllegalSQLFunctionArgument = errors.New("Illegal argument was used in the SQL function")
	37
	38	// ErrInvalidColumnIndex is an error if you provide a column index which is not
	39	// valid.
	40	var ErrInvalidColumnIndex = errors.New("Column index in the SQL expression is invalid")
	41
	42	// ErrInvalidKeyPath is an error if you provide a key in the SQL expression that
	43	// is invalid.
	44	var ErrInvalidKeyPath = errors.New("Key path in the SQL expression is invalid")
	45
	46	// ErrColumnTooLong is an error if your query results in a column that is
	47	// greater than the max amount of characters per column of 1mb
	48	var ErrColumnTooLong = errors.New("The length of a column in the result is greater than maxCharsPerColumn of 1 MB")
	49
	50	// ErrOverMaxColumn is an error if the number of columns from the resulting
	51	// query is greater than 1Mb.
	52	var ErrOverMaxColumn = errors.New("The number of columns in the result is greater than maxColumnNumber of 1 MB")
	53
	54	// ErrOverMaxRecordSize is an error if the length of a record in the result is
	55	// greater than 1 Mb.
	56	var ErrOverMaxRecordSize = errors.New("The length of a record in the result is greater than maxCharsPerRecord of 1 MB")
	57
	58	// ErrMissingHeaders is an error if some of the headers that are requested in
	59	// the Select Query are not present in the file.
	60	var ErrMissingHeaders = errors.New("Some headers in the query are missing from the file. Check the file and try again")
	61
	62	// ErrInvalidCompressionFormat is an error if an unsupported compression type is
	63	// utilized with the select object query.
	64	var ErrInvalidCompressionFormat = errors.New("The file is not in a supported compression format. Only GZIP is supported at this time")
	65
	66	// ErrTruncatedInput is an error if the object is not compressed properly and an
	67	// error occurs during decompression.
	68	var ErrTruncatedInput = errors.New("Object decompression failed. Check that the object is properly compressed using the format specified in the request")
	69
	70	// ErrInvalidFileHeaderInfo is an error if the argument provided to the
	71	// FileHeader Argument is incorrect.
	72	var ErrInvalidFileHeaderInfo = errors.New("The FileHeaderInfo is invalid. Only NONE, USE, and IGNORE are supported")
	73
	74	// ErrInvalidJSONType is an error if the json format provided as an argument is
	75	// invalid.
	76	var ErrInvalidJSONType = errors.New("The JsonType is invalid. Only DOCUMENT and LINES are supported at this time")
	77
	78	// ErrInvalidQuoteFields is an error if the arguments provided to the
	79	// QuoteFields options are not valid.
	80	var ErrInvalidQuoteFields = errors.New("The QuoteFields is invalid. Only ALWAYS and ASNEEDED are supported")
	81
	82	// ErrInvalidRequestParameter is an error if the value of a parameter in the
	83	// request element is not valid.
	84	var ErrInvalidRequestParameter = errors.New("The value of a parameter in Request element is invalid. Check the service API documentation and try again")
	85
	86	// ErrCSVParsingError is an error if the CSV presents an error while being
	87	// parsed.
	88	var ErrCSVParsingError = errors.New("Encountered an Error parsing the CSV file. Check the file and try again")
	89
	90	// ErrJSONParsingError is an error if while parsing the JSON an error arises.
	91	var ErrJSONParsingError = errors.New("Encountered an error parsing the JSON file. Check the file and try again")
	92
	93	// ErrExternalEvalException is an error that arises if the query can not be
	94	// evaluated.
	95	var ErrExternalEvalException = errors.New("The query cannot be evaluated. Check the file and try again")
	96
	97	// ErrInvalidDataType is an error that occurs if the SQL expression contains an
	98	// invalid data type.
	99	var ErrInvalidDataType = errors.New("The SQL expression contains an invalid data type")
	100
	101	// ErrUnrecognizedFormatException is an error that arises if there is an invalid
	102	// record type.
	103	var ErrUnrecognizedFormatException = errors.New("Encountered an invalid record type")
	104
	105	// ErrInvalidTextEncoding is an error if the text encoding is not valid.
	106	var ErrInvalidTextEncoding = errors.New("Invalid encoding type. Only UTF-8 encoding is supported at this time")
	107
	108	// ErrInvalidTableAlias is an error that arises if the table alias provided in
	109	// the SQL expression is invalid.
	110	var ErrInvalidTableAlias = errors.New("The SQL expression contains an invalid table alias")
	111
	112	// ErrMultipleDataSourcesUnsupported is an error that arises if multiple data
	113	// sources are provided.
	114	var ErrMultipleDataSourcesUnsupported = errors.New("Multiple data sources are not supported")
	115
	116	// ErrMissingRequiredParameter is an error that arises if a required argument
	117	// is omitted from the Request.
	118	var ErrMissingRequiredParameter = errors.New("The Request entity is missing a required parameter. Check the service documentation and try again")
	119
	120	// ErrObjectSerializationConflict is an error that arises if an unsupported
	121	// output seralization is provided.
	122	var ErrObjectSerializationConflict = errors.New("The Request entity can only contain one of CSV or JSON. Check the service documentation and try again")
	123
	124	// ErrUnsupportedSQLOperation is an error that arises if an unsupported SQL
	125	// operation is used.
	126	var ErrUnsupportedSQLOperation = errors.New("Encountered an unsupported SQL operation")
	127
	128	// ErrUnsupportedSQLStructure is an error that occurs if an unsupported SQL
	129	// structure is used.
	130	var ErrUnsupportedSQLStructure = errors.New("Encountered an unsupported SQL structure. Check the SQL Reference")
	131
	132	// ErrUnsupportedStorageClass is an error that occurs if an invalid storace
	133	// class is present.
	134	var ErrUnsupportedStorageClass = errors.New("Encountered an invalid storage class. Only STANDARD, STANDARD_IA, and ONEZONE_IA storage classes are supported at this time")
	135
	136	// ErrUnsupportedSyntax is an error that occurs if invalid syntax is present in
	137	// the query.
	138	var ErrUnsupportedSyntax = errors.New("Encountered invalid syntax")
	139
	140	// ErrUnsupportedRangeHeader is an error that occurs if a range header is
	141	// provided.
	142	var ErrUnsupportedRangeHeader = errors.New("Range header is not supported for this operation")
	143
	144	// ErrLexerInvalidChar is an error that occurs if the SQL expression contains an
	145	// invalid character.
	146	var ErrLexerInvalidChar = errors.New("The SQL expression contains an invalid character")
	147
	148	// ErrLexerInvalidOperator is an error that occurs if an invalid operator is
	149	// used.
	150	var ErrLexerInvalidOperator = errors.New("The SQL expression contains an invalid operator")
	151
	152	// ErrLexerInvalidLiteral is an error that occurs if an invalid literal is used.
	153	var ErrLexerInvalidLiteral = errors.New("The SQL expression contains an invalid literal")
	154
	155	// ErrLexerInvalidIONLiteral is an error that occurs if an invalid operator is
	156	// used
	157	var ErrLexerInvalidIONLiteral = errors.New("The SQL expression contains an invalid operator")
	158
	159	// ErrParseExpectedDatePart is an error that occurs if the date part is not
	160	// found in the SQL expression.
	161	var ErrParseExpectedDatePart = errors.New("Did not find the expected date part in the SQL expression")
	162
	163	// ErrParseExpectedKeyword is an error that occurs if the expected keyword was
	164	// not found in the expression.
	165	var ErrParseExpectedKeyword = errors.New("Did not find the expected keyword in the SQL expression")
	166
	167	// ErrParseExpectedTokenType is an error that occurs if the expected token is
	168	// not found in the SQL expression.
	169	var ErrParseExpectedTokenType = errors.New("Did not find the expected token in the SQL expression")
	170
	171	// ErrParseExpected2TokenTypes is an error that occurs if 2 token types are not
	172	// found.
	173	var ErrParseExpected2TokenTypes = errors.New("Did not find the expected token in the SQL expression")
	174
	175	// ErrParseExpectedNumber is an error that occurs if a number is expected but
	176	// not found in the expression.
	177	var ErrParseExpectedNumber = errors.New("Did not find the expected number in the SQL expression")
	178
	179	// ErrParseExpectedRightParenBuiltinFunctionCall is an error that occurs if a
	180	// right parenthesis is missing.
	181	var ErrParseExpectedRightParenBuiltinFunctionCall = errors.New("Did not find the expected right parenthesis character in the SQL expression")
	182
	183	// ErrParseExpectedTypeName is an error that occurs if a type name is expected
	184	// but not found.
	185	var ErrParseExpectedTypeName = errors.New("Did not find the expected type name in the SQL expression")
	186
	187	// ErrParseExpectedWhenClause is an error that occurs if a When clause is
	188	// expected but not found.
	189	var ErrParseExpectedWhenClause = errors.New("Did not find the expected WHEN clause in the SQL expression. CASE is not supported")
	190
	191	// ErrParseUnsupportedToken is an error that occurs if the SQL expression
	192	// contains an unsupported token.
	193	var ErrParseUnsupportedToken = errors.New("The SQL expression contains an unsupported token")
	194
	195	// ErrParseUnsupportedLiteralsGroupBy is an error that occurs if the SQL
	196	// expression has an unsupported use of Group By.
	197	var ErrParseUnsupportedLiteralsGroupBy = errors.New("The SQL expression contains an unsupported use of GROUP BY")
	198
	199	// ErrParseExpectedMember is an error that occurs if there is an unsupported use
	200	// of member in the SQL expression.
	201	var ErrParseExpectedMember = errors.New("The SQL expression contains an unsupported use of MEMBER")
	202
	203	// ErrParseUnsupportedSelect is an error that occurs if there is an unsupported
	204	// use of Select.
	205	var ErrParseUnsupportedSelect = errors.New("The SQL expression contains an unsupported use of SELECT")
	206
	207	// ErrParseUnsupportedCase is an error that occurs if there is an unsupported
	208	// use of case.
	209	var ErrParseUnsupportedCase = errors.New("The SQL expression contains an unsupported use of CASE")
	210
	211	// ErrParseUnsupportedCaseClause is an error that occurs if there is an
	212	// unsupported use of case.
	213	var ErrParseUnsupportedCaseClause = errors.New("The SQL expression contains an unsupported use of CASE")
	214
	215	// ErrParseUnsupportedAlias is an error that occurs if there is an unsupported
	216	// use of Alias.
	217	var ErrParseUnsupportedAlias = errors.New("The SQL expression contains an unsupported use of ALIAS")
	218
	219	// ErrParseUnsupportedSyntax is an error that occurs if there is an
	220	// UnsupportedSyntax in the SQL expression.
	221	var ErrParseUnsupportedSyntax = errors.New("The SQL expression contains unsupported syntax")
	222
	223	// ErrParseUnknownOperator is an error that occurs if there is an invalid
	224	// operator present in the SQL expression.
	225	var ErrParseUnknownOperator = errors.New("The SQL expression contains an invalid operator")
	226
	227	// ErrParseInvalidPathComponent is an error that occurs if there is an invalid
	228	// path component.
	229	var ErrParseInvalidPathComponent = errors.New("The SQL expression contains an invalid path component")
	230
	231	// ErrParseMissingIdentAfterAt is an error that occurs if the wrong symbol
	232	// follows the "@" symbol in the SQL expression.
	233	var ErrParseMissingIdentAfterAt = errors.New("Did not find the expected identifier after the @ symbol in the SQL expression")
	234
	235	// ErrParseUnexpectedOperator is an error that occurs if the SQL expression
	236	// contains an unexpected operator.
	237	var ErrParseUnexpectedOperator = errors.New("The SQL expression contains an unexpected operator")
	238
	239	// ErrParseUnexpectedTerm is an error that occurs if the SQL expression contains
	240	// an unexpected term.
	241	var ErrParseUnexpectedTerm = errors.New("The SQL expression contains an unexpected term")
	242
	243	// ErrParseUnexpectedToken is an error that occurs if the SQL expression
	244	// contains an unexpected token.
	245	var ErrParseUnexpectedToken = errors.New("The SQL expression contains an unexpected token")
	246
	247	// ErrParseUnexpectedKeyword is an error that occurs if the SQL expression
	248	// contains an unexpected keyword.
	249	var ErrParseUnexpectedKeyword = errors.New("The SQL expression contains an unexpected keyword")
	250
	251	// ErrParseExpectedExpression is an error that occurs if the SQL expression is
	252	// not found.
	253	var ErrParseExpectedExpression = errors.New("Did not find the expected SQL expression")
	254
	255	// ErrParseExpectedLeftParenAfterCast is an error that occurs if the left
	256	// parenthesis is missing after a cast in the SQL expression.
	257	var ErrParseExpectedLeftParenAfterCast = errors.New("Did not find the expected left parenthesis after CAST in the SQL expression")
	258
	259	// ErrParseExpectedLeftParenValueConstructor is an error that occurs if the left
	260	// parenthesis is not found in the SQL expression.
	261	var ErrParseExpectedLeftParenValueConstructor = errors.New("Did not find expected the left parenthesis in the SQL expression")
	262
	263	// ErrParseExpectedLeftParenBuiltinFunctionCall is an error that occurs if the
	264	// left parenthesis is not found in the SQL expression function call.
	265	var ErrParseExpectedLeftParenBuiltinFunctionCall = errors.New("Did not find the expected left parenthesis in the SQL expression")
	266
	267	// ErrParseExpectedArgumentDelimiter is an error that occurs if the argument
	268	// delimiter for the SQL expression is not provided.
	269	var ErrParseExpectedArgumentDelimiter = errors.New("Did not find the expected argument delimiter in the SQL expression")
	270
	271	// ErrParseCastArity is an error that occurs because the CAST has incorrect
	272	// arity.
	273	var ErrParseCastArity = errors.New("The SQL expression CAST has incorrect arity")
	274
	275	// ErrParseInvalidTypeParam is an error that occurs because there is an invalid
	276	// parameter value.
	277	var ErrParseInvalidTypeParam = errors.New("The SQL expression contains an invalid parameter value")
	278
	279	// ErrParseEmptySelect is an error that occurs because the SQL expression
	280	// contains an empty Select
	281	var ErrParseEmptySelect = errors.New("The SQL expression contains an empty SELECT")
	282
	283	// ErrParseSelectMissingFrom is an error that occurs because there is a missing
	284	// From after the Select List.
	285	var ErrParseSelectMissingFrom = errors.New("The SQL expression contains a missing FROM after SELECT list")
	286
	287	// ErrParseExpectedIdentForGroupName is an error that occurs because Group is
	288	// not supported in the SQL expression.
	289	var ErrParseExpectedIdentForGroupName = errors.New("GROUP is not supported in the SQL expression")
	290
	291	// ErrParseExpectedIdentForAlias is an error that occurs if expected identifier
	292	// for alias is not in the SQL expression.
	293	var ErrParseExpectedIdentForAlias = errors.New("Did not find the expected identifier for the alias in the SQL expression")
	294
	295	// ErrParseUnsupportedCallWithStar is an error that occurs if COUNT is used with
	296	// an argument other than "*".
	297	var ErrParseUnsupportedCallWithStar = errors.New("Only COUNT with (*) as a parameter is supported in the SQL expression")
	298
	299	// ErrParseNonUnaryAgregateFunctionCall is an error that occurs if more than one
	300	// argument is provided as an argument for aggregation functions.
	301	var ErrParseNonUnaryAgregateFunctionCall = errors.New("Only one argument is supported for aggregate functions in the SQL expression")
	302
	303	// ErrParseMalformedJoin is an error that occurs if a "join" operation is
	304	// attempted in the SQL expression as this is not supported.
	305	var ErrParseMalformedJoin = errors.New("JOIN is not supported in the SQL expression")
	306
	307	// ErrParseExpectedIdentForAt is an error that occurs if after "AT" an Alias
	308	// identifier is not provided.
	309	var ErrParseExpectedIdentForAt = errors.New("Did not find the expected identifier for AT name in the SQL expression")
	310
	311	// ErrParseAsteriskIsNotAloneInSelectList is an error that occurs if in addition
	312	// to an asterix, more column names are provided as arguments in the SQL
	313	// expression.
	314	var ErrParseAsteriskIsNotAloneInSelectList = errors.New("Other expressions are not allowed in the SELECT list when '*' is used without dot notation in the SQL expression")
	315
	316	// ErrParseCannotMixSqbAndWildcardInSelectList is an error that occurs if list
	317	// indexing and an asterix are mixed in the SQL expression.
	318	var ErrParseCannotMixSqbAndWildcardInSelectList = errors.New("Cannot mix [] and * in the same expression in a SELECT list in SQL expression")
	319
	320	// ErrParseInvalidContextForWildcardInSelectList is an error that occurs if the
	321	// asterix is used improperly within the SQL expression.
	322	var ErrParseInvalidContextForWildcardInSelectList = errors.New("Invalid use of * in SELECT list in the SQL expression")
	323
	324	// ErrEvaluatorBindingDoesNotExist is an error that occurs if a column name or
	325	// path provided in the expression does not exist.
	326	var ErrEvaluatorBindingDoesNotExist = errors.New("A column name or a path provided does not exist in the SQL expression")
	327
	328	// ErrIncorrectSQLFunctionArgumentType is an error that occurs if the wrong
	329	// argument is provided to a SQL function.
	330	var ErrIncorrectSQLFunctionArgumentType = errors.New("Incorrect type of arguments in function call in the SQL expression")
	331
	332	// ErrAmbiguousFieldName is an error that occurs if the column name which is not
	333	// case sensitive, is not descriptive enough to retrieve a singular column.
	334	var ErrAmbiguousFieldName = errors.New("Field name matches to multiple fields in the file. Check the SQL expression and the file, and try again")
	335
	336	// ErrEvaluatorInvalidArguments is an error that occurs if there are not the
	337	// correct number of arguments in a functional call to a SQL expression.
	338	var ErrEvaluatorInvalidArguments = errors.New("Incorrect number of arguments in the function call in the SQL expression")
	339
	340	// ErrValueParseFailure is an error that occurs if the Time Stamp is not parsed
	341	// correctly in the SQL expression.
	342	var ErrValueParseFailure = errors.New("Time stamp parse failure in the SQL expression")
	343
	344	// ErrIntegerOverflow is an error that occurs if there is an IntegerOverflow or
	345	// IntegerUnderFlow in the SQL expression.
	346	var ErrIntegerOverflow = errors.New("Int overflow or underflow in the SQL expression")
	347
	348	// ErrLikeInvalidInputs is an error that occurs if invalid inputs are provided
	349	// to the argument LIKE Clause.
	350	var ErrLikeInvalidInputs = errors.New("Invalid argument given to the LIKE clause in the SQL expression")
	351
	352	// ErrCastFailed occurs if the attempt to convert data types in the cast is not
	353	// done correctly.
	354	var ErrCastFailed = errors.New("Attempt to convert from one data type to another using CAST failed in the SQL expression")
	355
	356	// ErrInvalidCast is an error that occurs if the attempt to convert data types
	357	// failed and was done in an improper fashion.
	358	var ErrInvalidCast = errors.New("Attempt to convert from one data type to another using CAST failed in the SQL expression")
	359
	360	// ErrEvaluatorInvalidTimestampFormatPattern is an error that occurs if the Time
	361	// Stamp Format needs more additional fields to be filled.
	362	var ErrEvaluatorInvalidTimestampFormatPattern = errors.New("Time stamp format pattern requires additional fields in the SQL expression")
	363
	364	// ErrEvaluatorInvalidTimestampFormatPatternSymbolForParsing is an error that
	365	// occurs if the format of the time stamp can not be parsed.
	366	var ErrEvaluatorInvalidTimestampFormatPatternSymbolForParsing = errors.New("Time stamp format pattern contains a valid format symbol that cannot be applied to time stamp parsing in the SQL expression")
	367
	368	// ErrEvaluatorTimestampFormatPatternDuplicateFields is an error that occurs if
	369	// the time stamp format pattern contains multiple format specifications which
	370	// can not be clearly resolved.
	371	var ErrEvaluatorTimestampFormatPatternDuplicateFields = errors.New("Time stamp format pattern contains multiple format specifiers representing the time stamp field in the SQL expression")
	372
	373	//ErrEvaluatorTimestampFormatPatternHourClockAmPmMismatch is an error that
	374	//occurs if the time stamp format pattern contains a 12 hour day of format but
	375	//does not have an AM/PM field.
	376	var ErrEvaluatorTimestampFormatPatternHourClockAmPmMismatch = errors.New("Time stamp format pattern contains a 12-hour hour of day format symbol but doesn't also contain an AM/PM field, or it contains a 24-hour hour of day format specifier and contains an AM/PM field in the SQL expression")
	377
	378	// ErrEvaluatorUnterminatedTimestampFormatPatternToken is an error that occurs
	379	// if there is an unterminated token in the SQL expression for time stamp
	380	// format.
	381	var ErrEvaluatorUnterminatedTimestampFormatPatternToken = errors.New("Time stamp format pattern contains unterminated token in the SQL expression")
	382
	383	// ErrEvaluatorInvalidTimestampFormatPatternToken is an error that occurs if
	384	// there is an invalid token in the time stamp format within the SQL expression.
	385	var ErrEvaluatorInvalidTimestampFormatPatternToken = errors.New("Time stamp format pattern contains an invalid token in the SQL expression")
	386
	387	// ErrEvaluatorInvalidTimestampFormatPatternSymbol is an error that occurs if
	388	// the time stamp format pattern has an invalid symbol within the SQL
	389	// expression.
	390	var ErrEvaluatorInvalidTimestampFormatPatternSymbol = errors.New("Time stamp format pattern contains an invalid symbol in the SQL expression")
	391
	392	// S3 select API errors - TODO fix the errors.
	393	var errorCodeResponse = map[error]string{
	394	ErrBusy: "Busy",
	395	ErrUnauthorizedAccess: "UnauthorizedAccess",
	396	ErrExpressionTooLong: "ExpressionTooLong",
	397	ErrIllegalSQLFunctionArgument: "IllegalSqlFunctionArgument",
	398	ErrInvalidColumnIndex: "InvalidColumnIndex",
	399	ErrInvalidKeyPath: "InvalidKeyPath",
	400	ErrColumnTooLong: "ColumnTooLong",
	401	ErrOverMaxColumn: "OverMaxColumn",
	402	ErrOverMaxRecordSize: "OverMaxRecordSize",
	403	ErrMissingHeaders: "MissingHeaders",
	404	ErrInvalidCompressionFormat: "InvalidCompressionFormat",
	405	ErrTruncatedInput: "TruncatedInput",
	406	ErrInvalidFileHeaderInfo: "InvalidFileHeaderInfo",
	407	ErrInvalidJSONType: "InvalidJsonType",
	408	ErrInvalidQuoteFields: "InvalidQuoteFields",
	409	ErrInvalidRequestParameter: "InvalidRequestParameter",
	410	ErrCSVParsingError: "CSVParsingError",
	411	ErrJSONParsingError: "JSONParsingError",
	412	ErrExternalEvalException: "ExternalEvalException",
	413	ErrInvalidDataType: "InvalidDataType",
	414	ErrUnrecognizedFormatException: "UnrecognizedFormatException",
	415	ErrInvalidTextEncoding: "InvalidTextEncoding",
	416	ErrInvalidTableAlias: "InvalidTableAlias",
	417	ErrMultipleDataSourcesUnsupported: "MultipleDataSourcesUnsupported",
	418	ErrMissingRequiredParameter: "MissingRequiredParameter",
	419	ErrObjectSerializationConflict: "ObjectSerializationConflict",
	420	ErrUnsupportedSQLOperation: "UnsupportedSqlOperation",
	421	ErrUnsupportedSQLStructure: "UnsupportedSqlStructure",
	422	ErrUnsupportedStorageClass: "UnsupportedStorageClass",
	423	ErrUnsupportedSyntax: "UnsupportedSyntax",
	424	ErrUnsupportedRangeHeader: "UnsupportedRangeHeader",
	425	ErrLexerInvalidChar: "LexerInvalidChar",
	426	ErrLexerInvalidOperator: "LexerInvalidOperator",
	427	ErrLexerInvalidLiteral: "LexerInvalidLiteral",
	428	ErrLexerInvalidIONLiteral: "LexerInvalidIONLiteral",
	429	ErrParseExpectedDatePart: "ParseExpectedDatePart",
	430	ErrParseExpectedKeyword: "ParseExpectedKeyword",
	431	ErrParseExpectedTokenType: "ParseExpectedTokenType",
	432	ErrParseExpected2TokenTypes: "ParseExpected2TokenTypes",
	433	ErrParseExpectedNumber: "ParseExpectedNumber",
	434	ErrParseExpectedRightParenBuiltinFunctionCall: "ParseExpectedRightParenBuiltinFunctionCall",
	435	ErrParseExpectedTypeName: "ParseExpectedTypeName",
	436	ErrParseExpectedWhenClause: "ParseExpectedWhenClause",
	437	ErrParseUnsupportedToken: "ParseUnsupportedToken",
	438	ErrParseUnsupportedLiteralsGroupBy: "ParseUnsupportedLiteralsGroupBy",
	439	ErrParseExpectedMember: "ParseExpectedMember",
	440	ErrParseUnsupportedSelect: "ParseUnsupportedSelect",
	441	ErrParseUnsupportedCase: "ParseUnsupportedCase:",
	442	ErrParseUnsupportedCaseClause: "ParseUnsupportedCaseClause",
	443	ErrParseUnsupportedAlias: "ParseUnsupportedAlias",
	444	ErrParseUnsupportedSyntax: "ParseUnsupportedSyntax",
	445	ErrParseUnknownOperator: "ParseUnknownOperator",
	446	ErrParseInvalidPathComponent: "ParseInvalidPathComponent",
	447	ErrParseMissingIdentAfterAt: "ParseMissingIdentAfterAt",
	448	ErrParseUnexpectedOperator: "ParseUnexpectedOperator",
	449	ErrParseUnexpectedTerm: "ParseUnexpectedTerm",
	450	ErrParseUnexpectedToken: "ParseUnexpectedToken",
	451	ErrParseUnexpectedKeyword: "ParseUnexpectedKeyword",
	452	ErrParseExpectedExpression: "ParseExpectedExpression",
	453	ErrParseExpectedLeftParenAfterCast: "ParseExpectedLeftParenAfterCast",
	454	ErrParseExpectedLeftParenValueConstructor: "ParseExpectedLeftParenValueConstructor",
	455	ErrParseExpectedLeftParenBuiltinFunctionCall: "ParseExpectedLeftParenBuiltinFunctionCall",
	456	ErrParseExpectedArgumentDelimiter: "ParseExpectedArgumentDelimiter",
	457	ErrParseCastArity: "ParseCastArity",
	458	ErrParseInvalidTypeParam: "ParseInvalidTypeParam",
	459	ErrParseEmptySelect: "ParseEmptySelect",
	460	ErrParseSelectMissingFrom: "ParseSelectMissingFrom",
	461	ErrParseExpectedIdentForGroupName: "ParseExpectedIdentForGroupName",
	462	ErrParseExpectedIdentForAlias: "ParseExpectedIdentForAlias",
	463	ErrParseUnsupportedCallWithStar: "ParseUnsupportedCallWithStar",
	464	ErrParseNonUnaryAgregateFunctionCall: "ParseNonUnaryAgregateFunctionCall",
	465	ErrParseMalformedJoin: "ParseMalformedJoin",
	466	ErrParseExpectedIdentForAt: "ParseExpectedIdentForAt",
	467	ErrParseAsteriskIsNotAloneInSelectList: "ParseAsteriskIsNotAloneInSelectList",
	468	ErrParseCannotMixSqbAndWildcardInSelectList: "ParseCannotMixSqbAndWildcardInSelectList",
	469	ErrParseInvalidContextForWildcardInSelectList: "ParseInvalidContextForWildcardInSelectList",
	470	ErrEvaluatorBindingDoesNotExist: "EvaluatorBindingDoesNotExist",
	471	ErrIncorrectSQLFunctionArgumentType: "IncorrectSqlFunctionArgumentType",
	472	ErrAmbiguousFieldName: "AmbiguousFieldName",
	473	ErrEvaluatorInvalidArguments: "EvaluatorInvalidArguments",
	474	ErrValueParseFailure: "ValueParseFailure",
	475	ErrIntegerOverflow: "IntegerOverflow",
	476	ErrLikeInvalidInputs: "LikeInvalidInputs",
	477	ErrCastFailed: "CastFailed",
	478	ErrInvalidCast: "Attempt to convert from one data type to another using CAST failed in the SQL expression.",
	479	ErrEvaluatorInvalidTimestampFormatPattern: "EvaluatorInvalidTimestampFormatPattern",
	480	ErrEvaluatorInvalidTimestampFormatPatternSymbolForParsing: "EvaluatorInvalidTimestampFormatPatternSymbolForParsing",
	481	ErrEvaluatorTimestampFormatPatternDuplicateFields: "EvaluatorTimestampFormatPatternDuplicateFields",
	482	ErrEvaluatorTimestampFormatPatternHourClockAmPmMismatch: "EvaluatorTimestampFormatPatternHourClockAmPmMismatch",
	483	ErrEvaluatorUnterminatedTimestampFormatPatternToken: "EvaluatorUnterminatedTimestampFormatPatternToken",
	484	ErrEvaluatorInvalidTimestampFormatPatternToken: "EvaluatorInvalidTimestampFormatPatternToken",
	485	ErrEvaluatorInvalidTimestampFormatPatternSymbol: "EvaluatorInvalidTimestampFormatPatternSymbol",
	486	}

pkg/s3select/funcEval.go+231−0

@@ -0,0 +1,231 @@
	1	/*
	2	* Minio Cloud Storage, (C) 2018 Minio, Inc.
	3	*
	4	* Licensed under the Apache License, Version 2.0 (the "License");
	5	* you may not use this file except in compliance with the License.
	6	* You may obtain a copy of the License at
	7	*
	8	* http://www.apache.org/licenses/LICENSE-2.0
	9	*
	10	* Unless required by applicable law or agreed to in writing, software
	11	* distributed under the License is distributed on an "AS IS" BASIS,
	12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	* See the License for the specific language governing permissions and
	14	* limitations under the License.
	15	*/
	16
	17	package s3select
	18
	19	import (
	20	"strings"
	21
	22	"github.com/xwb1989/sqlparser"
	23	)
	24
	25	// stringOps is a function which handles the case in a clause if there is a need
	26	// to perform a string function
	27	func stringOps(myFunc *sqlparser.FuncExpr, record []string, myReturnVal string, columnsMap map[string]int) string {
	28	var value string
	29	funcName := myFunc.Name.CompliantName()
	30	switch tempArg := myFunc.Exprs[0].(type) {
	31	case *sqlparser.AliasedExpr:
	32	switch col := tempArg.Expr.(type) {
	33	case *sqlparser.FuncExpr:
	34	// myReturnVal is actually the tail recursive value being used in the eval func.
	35	return applyStrFunc(myReturnVal, funcName)
	36	case *sqlparser.ColName:
	37	value = applyStrFunc(record[columnsMap[col.Name.CompliantName()]], funcName)
	38	case *sqlparser.SQLVal:
	39	value = applyStrFunc(string(col.Val), funcName)
	40	}
	41	}
	42	return value
	43	}
	44
	45	// coalOps is a function which decomposes a COALESCE func expr into its struct.
	46	func coalOps(myFunc *sqlparser.FuncExpr, record []string, myReturnVal string, columnsMap map[string]int) string {
	47	myArgs := make([]string, len(myFunc.Exprs))
	48
	49	for i := 0; i < len(myFunc.Exprs); i++ {
	50	switch tempArg := myFunc.Exprs[i].(type) {
	51	case *sqlparser.AliasedExpr:
	52	switch col := tempArg.Expr.(type) {
	53	case *sqlparser.FuncExpr:
	54	// myReturnVal is actually the tail recursive value being used in the eval func.
	55	return myReturnVal
	56	case *sqlparser.ColName:
	57	myArgs[i] = record[columnsMap[col.Name.CompliantName()]]
	58	case *sqlparser.SQLVal:
	59	myArgs[i] = string(col.Val)
	60	}
	61	}
	62	}
	63	return processCoalNoIndex(myArgs)
	64	}
	65
	66	// nullOps is a function which decomposes a NullIf func expr into its struct.
	67	func nullOps(myFunc *sqlparser.FuncExpr, record []string, myReturnVal string, columnsMap map[string]int) string {
	68	myArgs := make([]string, 2)
	69
	70	for i := 0; i < len(myFunc.Exprs); i++ {
	71	switch tempArg := myFunc.Exprs[i].(type) {
	72	case *sqlparser.AliasedExpr:
	73	switch col := tempArg.Expr.(type) {
	74	case *sqlparser.FuncExpr:
	75	return myReturnVal
	76	case *sqlparser.ColName:
	77	myArgs[i] = record[columnsMap[col.Name.CompliantName()]]
	78	case *sqlparser.SQLVal:
	79	myArgs[i] = string(col.Val)
	80	}
	81	}
	82	}
	83	return processNullIf(myArgs)
	84	}
	85
	86	// isValidString is a function that ensures the current index is one with a
	87	// StrFunc
	88	func isValidFunc(myList []int, index int) bool {
	89	if myList == nil {
	90	return false
	91	}
	92	for i := 0; i < len(myList); i++ {
	93	if myList[i] == index {
	94	return true
	95	}
	96	}
	97	return false
	98	}
	99
	100	// processNullIf is a function that evaluates a given NULLIF clause.
	101	func processNullIf(nullStore []string) string {
	102	nullValOne := nullStore[0]
	103	nullValTwo := nullStore[1]
	104	if nullValOne == nullValTwo {
	105	return ""
	106	}
	107	return nullValOne
	108	}
	109
	110	// processCoalNoIndex is a function which evaluates a given COALESCE clause.
	111	func processCoalNoIndex(coalStore []string) string {
	112	for i := 0; i < len(coalStore); i++ {
	113	if coalStore[i] != "null" && coalStore[i] != "missing" && coalStore[i] != "" {
	114	return coalStore[i]
	115	}
	116	}
	117	return "null"
	118	}
	119
	120	// evaluateFuncExpr is a function that allows for tail recursive evaluation of
	121	// nested function expressions.
	122	func evaluateFuncExpr(myVal *sqlparser.FuncExpr, myReturnVal string, myRecord []string, columnsMap map[string]int) string {
	123	if myVal == nil {
	124	return myReturnVal
	125	}
	126	// retrieve all the relevant arguments of the function
	127	var mySubFunc []*sqlparser.FuncExpr
	128	mySubFunc = make([]*sqlparser.FuncExpr, len(myVal.Exprs))
	129	for i := 0; i < len(myVal.Exprs); i++ {
	130	switch col := myVal.Exprs[i].(type) {
	131	case *sqlparser.AliasedExpr:
	132	switch temp := col.Expr.(type) {
	133	case *sqlparser.FuncExpr:
	134	mySubFunc[i] = temp
	135	}
	136	}
	137	}
	138	// Need to do tree recursion so as to explore all possible directions of the
	139	// nested function recursion
	140	for i := 0; i < len(mySubFunc); i++ {
	141	if supportedString(myVal.Name.CompliantName()) {
	142	if mySubFunc != nil {
	143	return stringOps(myVal, myRecord, evaluateFuncExpr(mySubFunc[i], myReturnVal, myRecord, columnsMap), columnsMap)
	144	}
	145	return stringOps(myVal, myRecord, myReturnVal, columnsMap)
	146	} else if strings.ToUpper(myVal.Name.CompliantName()) == "NULLIF" {
	147	if mySubFunc != nil {
	148	return nullOps(myVal, myRecord, evaluateFuncExpr(mySubFunc[i], myReturnVal, myRecord, columnsMap), columnsMap)
	149	}
	150	return nullOps(myVal, myRecord, myReturnVal, columnsMap)
	151	} else if strings.ToUpper(myVal.Name.CompliantName()) == "COALESCE" {
	152	if mySubFunc != nil {
	153	return coalOps(myVal, myRecord, evaluateFuncExpr(mySubFunc[i], myReturnVal, myRecord, columnsMap), columnsMap)
	154	}
	155	return coalOps(myVal, myRecord, myReturnVal, columnsMap)
	156	}
	157	}
	158	return ""
	159	}
	160
	161	// evaluateFuncErr is a function that flags errors in nested functions.
	162	func (reader Input) evaluateFuncErr(myVal sqlparser.FuncExpr) error {
	163	if myVal == nil {
	164	return nil
	165	}
	166	if !supportedFunc(myVal.Name.CompliantName()) {
	167	return ErrUnsupportedSQLOperation
	168	}
	169	for i := 0; i < len(myVal.Exprs); i++ {
	170	switch tempArg := myVal.Exprs[i].(type) {
	171	case *sqlparser.StarExpr:
	172	return ErrParseUnsupportedCallWithStar
	173	case *sqlparser.AliasedExpr:
	174	switch col := tempArg.Expr.(type) {
	175	case *sqlparser.FuncExpr:
	176	if err := reader.evaluateFuncErr(col); err != nil {
	177	return err
	178	}
	179	case *sqlparser.ColName:
	180	if err := reader.colNameErrs([]string{col.Name.CompliantName()}); err != nil {
	181	return err
	182	}
	183	}
	184	}
	185	}
	186	return nil
	187	}
	188
	189	// evaluateIsExpr is a function for evaluating expressions of the form "column
	190	// is ...."
	191	func evaluateIsExpr(myFunc *sqlparser.IsExpr, row []string, columnNames map[string]int, alias string) (bool, error) {
	192	operator := myFunc.Operator
	193	var colName string
	194	var myVal string
	195	switch myIs := myFunc.Expr.(type) {
	196	// case for literal val
	197	case *sqlparser.SQLVal:
	198	myVal = string(myIs.Val)
	199	// case for nested func val
	200	case *sqlparser.FuncExpr:
	201	myVal = evaluateFuncExpr(myIs, "", row, columnNames)
	202	// case for col val
	203	case *sqlparser.ColName:
	204	colName = cleanCol(myIs.Name.CompliantName(), alias)
	205	}
	206	// case if it is a col val
	207	if colName != "" {
	208	myVal = row[columnNames[colName]]
	209	}
	210	// case to evaluate is null
	211	if strings.ToLower(operator) == "is null" {
	212	return myVal == "", nil
	213	}
	214	// case to evaluate is not null
	215	if strings.ToLower(operator) == "is not null" {
	216	return myVal != "", nil
	217	}
	218	return false, ErrUnsupportedSQLOperation
	219	}
	220
	221	// supportedString is a function that checks whether the function is a supported
	222	// string one
	223	func supportedString(strFunc string) bool {
	224	return stringInSlice(strings.ToUpper(strFunc), []string{"TRIM", "SUBSTRING", "CHAR_LENGTH", "CHARACTER_LENGTH", "LOWER", "UPPER"})
	225	}
	226
	227	// supportedFunc is a function that checks whether the function is a supported
	228	// S3 one.
	229	func supportedFunc(strFunc string) bool {
	230	return stringInSlice(strings.ToUpper(strFunc), []string{"TRIM", "SUBSTRING", "CHAR_LENGTH", "CHARACTER_LENGTH", "LOWER", "UPPER", "COALESCE", "NULLIF"})
	231	}

pkg/s3select/helpers.go+754−0

@@ -0,0 +1,754 @@
	1	/*
	2	* Minio Cloud Storage, (C) 2018 Minio, Inc.
	3	*
	4	* Licensed under the Apache License, Version 2.0 (the "License");
	5	* you may not use this file except in compliance with the License.
	6	* You may obtain a copy of the License at
	7	*
	8	* http://www.apache.org/licenses/LICENSE-2.0
	9	*
	10	* Unless required by applicable law or agreed to in writing, software
	11	* distributed under the License is distributed on an "AS IS" BASIS,
	12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	* See the License for the specific language governing permissions and
	14	* limitations under the License.
	15	*/
	16
	17	package s3select
	18
	19	import (
	20	"fmt"
	21	"reflect"
	22	"strconv"
	23	"strings"
	24
	25	"github.com/xwb1989/sqlparser"
	26	)
	27
	28	// This function processes size so that we can calculate bytes BytesProcessed.
	29	func processSize(myrecord []string) int64 {
	30	if len(myrecord) > 0 {
	31	var size int64
	32	size = int64(len(myrecord)-1) + 1
	33	for i := range myrecord {
	34	size += int64(len(myrecord[i]))
	35	}
	36
	37	return size
	38	}
	39	return 0
	40	}
	41
	42	// This function finds whether a string is in a list
	43	func stringInSlice(x string, list []string) bool {
	44	for _, y := range list {
	45	if x == y {
	46	return true
	47	}
	48	}
	49	return false
	50	}
	51
	52	// This function returns the index of a string in a list
	53	func stringIndex(a string, list []string) int {
	54	for i := range list {
	55	if list[i] == a {
	56	return i
	57	}
	58	}
	59	return -1
	60	}
	61
	62	// Returns a true or false, whether a string can be represented as an int.
	63	func representsInt(s string) bool {
	64	if _, err := strconv.Atoi(s); err == nil {
	65	return true
	66	}
	67	return false
	68	}
	69
	70	// The function below processes the where clause into an acutal boolean given a
	71	// row
	72	func matchesMyWhereClause(row []string, columnNames map[string]int, alias string, whereClause interface{}) (bool, error) {
	73	// This particular logic deals with the details of casting, e.g if we have to
	74	// cast a column of string numbers into int's for comparison.
	75	var conversionColumn string
	76	var operator string
	77	var operand interface{}
	78	if fmt.Sprintf("%v", whereClause) == "false" {
	79	return false, nil
	80	}
	81	switch expr := whereClause.(type) {
	82	case *sqlparser.IsExpr:
	83	return evaluateIsExpr(expr, row, columnNames, alias)
	84	case *sqlparser.RangeCond:
	85	operator = expr.Operator
	86	if operator != "between" && operator != "not between" {
	87	return false, ErrUnsupportedSQLOperation
	88	}
	89	if operator == "not between" {
	90	myResult, err := evaluateBetween(expr, alias, row, columnNames)
	91	if err != nil {
	92	return false, err
	93	}
	94	return !myResult, nil
	95	}
	96	myResult, err := evaluateBetween(expr, alias, row, columnNames)
	97	if err != nil {
	98	return false, err
	99	}
	100	return myResult, nil
	101	case *sqlparser.ComparisonExpr:
	102	operator = expr.Operator
	103	switch right := expr.Right.(type) {
	104	case *sqlparser.FuncExpr:
	105	operand = evaluateFuncExpr(right, "", row, columnNames)
	106	case *sqlparser.SQLVal:
	107	var err error
	108	operand, err = evaluateParserType(right)
	109	if err != nil {
	110	return false, err
	111	}
	112	}
	113	var myVal string
	114	myVal = ""
	115	switch left := expr.Left.(type) {
	116	case *sqlparser.FuncExpr:
	117	myVal = evaluateFuncExpr(left, "", row, columnNames)
	118	conversionColumn = ""
	119	case *sqlparser.ColName:
	120	conversionColumn = cleanCol(left.Name.CompliantName(), alias)
	121	}
	122	if representsInt(conversionColumn) {
	123	intCol, err := strconv.Atoi(conversionColumn)
	124	if err != nil {
	125	return false, err
	126	}
	127	// Subtract 1 out because the index starts at 1 for Amazon instead of 0.
	128	return evaluateOperator(row[intCol-1], operator, operand)
	129	}
	130	if myVal != "" {
	131	return evaluateOperator(myVal, operator, operand)
	132	}
	133	return evaluateOperator(row[columnNames[conversionColumn]], operator, operand)
	134	case *sqlparser.AndExpr:
	135	var leftVal bool
	136	var rightVal bool
	137	switch left := expr.Left.(type) {
	138	case *sqlparser.ComparisonExpr:
	139	temp, err := matchesMyWhereClause(row, columnNames, alias, left)
	140	if err != nil {
	141	return false, err
	142	}
	143	leftVal = temp
	144	}
	145	switch right := expr.Right.(type) {
	146	case *sqlparser.ComparisonExpr:
	147	temp, err := matchesMyWhereClause(row, columnNames, alias, right)
	148	if err != nil {
	149	return false, err
	150	}
	151	rightVal = temp
	152	}
	153	return (rightVal && leftVal), nil
	154	case *sqlparser.OrExpr:
	155	var leftVal bool
	156	var rightVal bool
	157	switch left := expr.Left.(type) {
	158	case *sqlparser.ComparisonExpr:
	159	leftVal, _ = matchesMyWhereClause(row, columnNames, alias, left)
	160
	161	}
	162	switch right := expr.Right.(type) {
	163	case *sqlparser.ComparisonExpr:
	164	rightVal, _ = matchesMyWhereClause(row, columnNames, alias, right)
	165	}
	166	return (rightVal \|\| leftVal), nil
	167
	168	}
	169	return true, nil
	170	}
	171	func applyStrFunc(rawArg string, funcName string) string {
	172	switch strings.ToUpper(funcName) {
	173	case "TRIM":
	174	// parser has an issue which does not allow it to support Trim with other
	175	// arguments
	176	return strings.Trim(rawArg, " ")
	177	case "SUBSTRING":
	178	// TODO parser has an issue which does not support substring
	179	return rawArg
	180	case "CHAR_LENGTH":
	181	return strconv.Itoa(len(rawArg))
	182	case "CHARACTER_LENGTH":
	183	return strconv.Itoa(len(rawArg))
	184	case "LOWER":
	185	return strings.ToLower(rawArg)
	186	case "UPPER":
	187	return strings.ToUpper(rawArg)
	188	}
	189	return rawArg
	190
	191	}
	192
	193	// This is a really important function it actually evaluates the boolean
	194	// statement and therefore actually returns a bool, it functions as the lowest
	195	// level of the state machine.
	196	func evaluateOperator(myTblVal string, operator string, operand interface{}) (bool, error) {
	197	if err := checkValidOperator(operator); err != nil {
	198	return false, err
	199	}
	200	myRecordVal := checkStringType(myTblVal)
	201	myVal := reflect.ValueOf(myRecordVal)
	202	myOp := reflect.ValueOf(operand)
	203
	204	switch {
	205	case myVal.Kind() == reflect.String && myOp.Kind() == reflect.String:
	206	return stringEval(myVal.String(), operator, myOp.String())
	207	case myVal.Kind() == reflect.Float64 && myOp.Kind() == reflect.Float64:
	208	return floatEval(myVal.Float(), operator, myOp.Float())
	209	case myVal.Kind() == reflect.Int && myOp.Kind() == reflect.Int:
	210	return intEval(myVal.Int(), operator, myOp.Int())
	211	case myVal.Kind() == reflect.Int && myOp.Kind() == reflect.String:
	212	stringVs := strconv.Itoa(int(myVal.Int()))
	213	return stringEval(stringVs, operator, myOp.String())
	214	case myVal.Kind() == reflect.Float64 && myOp.Kind() == reflect.String:
	215	stringVs := strconv.FormatFloat(myVal.Float(), 'f', 6, 64)
	216	return stringEval(stringVs, operator, myOp.String())
	217	case myVal.Kind() != myOp.Kind():
	218	return false, nil
	219	}
	220	return false, ErrUnsupportedSyntax
	221	}
	222
	223	// checkValidOperator ensures that the current operator is supported
	224	func checkValidOperator(operator string) error {
	225	listOfOps := []string{">", "<", "=", "<=", ">=", "!=", "like"}
	226	for i := range listOfOps {
	227	if operator == listOfOps[i] {
	228	return nil
	229	}
	230	}
	231	return ErrParseUnknownOperator
	232	}
	233
	234	// checkStringType converts the value from the csv to the appropriate one.
	235	func checkStringType(myTblVal string) interface{} {
	236	myInt, isInt := strconv.Atoi(myTblVal)
	237	myFloat, isFloat := strconv.ParseFloat(myTblVal, 64)
	238	if isInt == nil {
	239	return myInt
	240	} else if isFloat == nil {
	241	return myFloat
	242	} else {
	243	return myTblVal
	244	}
	245	}
	246
	247	// stringEval is for evaluating the state of string comparison.
	248	func stringEval(myRecordVal string, operator string, myOperand string) (bool, error) {
	249	switch operator {
	250	case ">":
	251	return myRecordVal > myOperand, nil
	252	case "<":
	253	return myRecordVal < myOperand, nil
	254	case "=":
	255	return myRecordVal == myOperand, nil
	256	case "<=":
	257	return myRecordVal <= myOperand, nil
	258	case ">=":
	259	return myRecordVal >= myOperand, nil
	260	case "!=":
	261	return myRecordVal != myOperand, nil
	262	case "like":
	263	return likeConvert(myOperand, myRecordVal)
	264	}
	265	return false, ErrUnsupportedSyntax
	266	}
	267
	268	// intEval is for evaluating integer comparisons.
	269	func intEval(myRecordVal int64, operator string, myOperand int64) (bool, error) {
	270
	271	switch operator {
	272	case ">":
	273	return myRecordVal > myOperand, nil
	274	case "<":
	275	return myRecordVal < myOperand, nil
	276	case "=":
	277	return myRecordVal == myOperand, nil
	278	case "<=":
	279	return myRecordVal <= myOperand, nil
	280	case ">=":
	281	return myRecordVal >= myOperand, nil
	282	case "!=":
	283	return myRecordVal != myOperand, nil
	284	}
	285	return false, ErrUnsupportedSyntax
	286	}
	287
	288	// floatEval is for evaluating the comparison of floats.
	289	func floatEval(myRecordVal float64, operator string, myOperand float64) (bool, error) {
	290	// Basically need some logic thats like, if the types dont match check for a cast
	291	switch operator {
	292	case ">":
	293	return myRecordVal > myOperand, nil
	294	case "<":
	295	return myRecordVal < myOperand, nil
	296	case "=":
	297	return myRecordVal == myOperand, nil
	298	case "<=":
	299	return myRecordVal <= myOperand, nil
	300	case ">=":
	301	return myRecordVal >= myOperand, nil
	302	case "!=":
	303	return myRecordVal != myOperand, nil
	304	}
	305	return false, ErrUnsupportedSyntax
	306	}
	307
	308	// prefixMatch allows for matching a prefix only like query e.g a%
	309	func prefixMatch(pattern string, record string) bool {
	310	for i := 0; i < len(pattern)-1; i++ {
	311	if pattern[i] != record[i] && pattern[i] != byte('_') {
	312	return false
	313	}
	314	}
	315	return true
	316	}
	317
	318	// suffixMatch allows for matching a suffix only like query e.g %an
	319	func suffixMatch(pattern string, record string) bool {
	320	for i := len(pattern) - 1; i > 0; i-- {
	321	if pattern[i] != record[len(record)-(len(pattern)-i)] && pattern[i] != byte('_') {
	322	return false
	323	}
	324	}
	325	return true
	326	}
	327
	328	// This function is for evaluating select statements which are case sensitive
	329	func likeConvert(pattern string, record string) (bool, error) {
	330	// If pattern is empty just return false
	331	if pattern == "" \|\| record == "" {
	332	return false, nil
	333	}
	334	// for suffix match queries e.g %a
	335	if len(pattern) >= 2 && pattern[0] == byte('%') && strings.Count(pattern, "%") == 1 {
	336	return suffixMatch(pattern, record), nil
	337	}
	338	// for prefix match queries e.g a%
	339	if len(pattern) >= 2 && pattern[len(pattern)-1] == byte('%') && strings.Count(pattern, "%") == 1 {
	340	return prefixMatch(pattern, record), nil
	341	}
	342	charCount := 0
	343	currPos := 0
	344	// Loop through the pattern so that a boolean can be returned
	345	for i := 0; i < len(pattern); i++ {
	346	if pattern[i] == byte('_') {
	347	// if its an underscore it can be anything so shift current position for
	348	// pattern and string
	349	charCount++
	350	// if there have been more characters in the pattern than record, clearly
	351	// there should be a return
	352	if i != len(pattern)-1 {
	353	if pattern[i+1] != byte('%') && pattern[i+1] != byte('_') {
	354	if currPos != len(record)-1 && pattern[i+1] != record[currPos+1] {
	355	return false, nil
	356	}
	357	}
	358	}
	359	if charCount > len(record) {
	360	return false, nil
	361	}
	362	// if the pattern has been fully evaluated, then just return.
	363	if len(pattern) == i+1 {
	364	return true, nil
	365	}
	366	i++
	367	currPos++
	368	}
	369	if pattern[i] == byte('%') \|\| pattern[i] == byte('*') {
	370	// if there is a wildcard then want to return true if its last and flag it.
	371	if currPos == len(record) {
	372	return false, nil
	373	}
	374	if i+1 == len(pattern) {
	375	return true, nil
	376	}
	377	} else {
	378	charCount++
	379	matched := false
	380	// iterate through the pattern and check if there is a match for the
	381	// character
	382	for currPos < len(record) {
	383	if record[currPos] == pattern[i] \|\| pattern[i] == byte('_') {
	384	matched = true
	385	break
	386	}
	387	currPos++
	388	}
	389	currPos++
	390	// if the character did not match then return should occur.
	391	if !matched {
	392	return false, nil
	393	}
	394	}
	395	}
	396	if charCount > len(record) {
	397	return false, nil
	398	}
	399	if currPos < len(record) {
	400	return false, nil
	401	}
	402	return true, nil
	403	}
	404
	405	// TrimQuotes allows the following to occur select "name", we need to trim the
	406	// quotes to reference our map of columnNames.
	407	func trimQuotes(s string) string {
	408	if len(s) >= 2 {
	409	if c := s[len(s)-1]; s[0] == c && (c == '"') {
	410	return s[1 : len(s)-1]
	411	}
	412	}
	413	return s
	414	}
	415
	416	// cleanCol cleans a column name from the parser so that the name is returned to
	417	// original.
	418	func cleanCol(myCol string, alias string) string {
	419	if len(myCol) <= 0 {
	420	return myCol
	421	}
	422	if !strings.HasPrefix(myCol, alias) && myCol[0] == '_' {
	423	myCol = alias + myCol
	424	}
	425
	426	if strings.Contains(myCol, ".") {
	427	myCol = strings.Replace(myCol, alias+"._", "", len(myCol))
	428	}
	429	myCol = strings.Replace(myCol, alias+"_", "", len(myCol))
	430	return myCol
	431	}
	432
	433	// evaluateBetween is a function which evaluates a Between Clause.
	434	func evaluateBetween(betweenExpr *sqlparser.RangeCond, alias string, record []string, columnNames map[string]int) (bool, error) {
	435	var colToVal interface{}
	436	var colFromVal interface{}
	437	var conversionColumn string
	438	var funcName string
	439	switch colTo := betweenExpr.To.(type) {
	440	case sqlparser.Expr:
	441	switch colToMyVal := colTo.(type) {
	442	case *sqlparser.FuncExpr:
	443	var temp string
	444	temp = stringOps(colToMyVal, record, "", columnNames)
	445	colToVal = []byte(temp)
	446	case *sqlparser.SQLVal:
	447	var err error
	448	colToVal, err = evaluateParserType(colToMyVal)
	449	if err != nil {
	450	return false, err
	451	}
	452	}
	453	}
	454	switch colFrom := betweenExpr.From.(type) {
	455	case sqlparser.Expr:
	456	switch colFromMyVal := colFrom.(type) {
	457	case *sqlparser.FuncExpr:
	458	colFromVal = stringOps(colFromMyVal, record, "", columnNames)
	459	case *sqlparser.SQLVal:
	460	var err error
	461	colFromVal, err = evaluateParserType(colFromMyVal)
	462	if err != nil {
	463	return false, err
	464	}
	465	}
	466	}
	467	var myFuncVal string
	468	myFuncVal = ""
	469	switch left := betweenExpr.Left.(type) {
	470	case *sqlparser.FuncExpr:
	471	myFuncVal = evaluateFuncExpr(left, "", record, columnNames)
	472	conversionColumn = ""
	473	case *sqlparser.ColName:
	474	conversionColumn = cleanCol(left.Name.CompliantName(), alias)
	475	}
	476
	477	toGreater, err := evaluateOperator(fmt.Sprintf("%v", colToVal), ">", colFromVal)
	478	if err != nil {
	479	return false, err
	480	}
	481	if toGreater {
	482	return evalBetweenGreater(conversionColumn, record, funcName, columnNames, colFromVal, colToVal, myFuncVal)
	483	}
	484	return evalBetweenLess(conversionColumn, record, funcName, columnNames, colFromVal, colToVal, myFuncVal)
	485	}
	486
	487	// evalBetweenLess is a function which evaluates the between given that the
	488	// FROM is > than the TO.
	489	func evalBetweenLess(conversionColumn string, record []string, funcName string, columnNames map[string]int, colFromVal interface{}, colToVal interface{}, myCoalVal string) (bool, error) {
	490	if representsInt(conversionColumn) {
	491	myIndex, _ := strconv.Atoi(conversionColumn)
	492	// Subtract 1 out because the index starts at 1 for Amazon instead of 0.
	493	myVal, err := evaluateOperator(record[myIndex-1], "<=", colFromVal)
	494	if err != nil {
	495	return false, err
	496	}
	497	var myOtherVal bool
	498	myOtherVal, err = evaluateOperator(fmt.Sprintf("%v", colToVal), "<=", checkStringType(record[myIndex-1]))
	499	if err != nil {
	500	return false, err
	501	}
	502	return (myVal && myOtherVal), nil
	503	}
	504	if myCoalVal != "" {
	505	myVal, err := evaluateOperator(myCoalVal, "<=", colFromVal)
	506	if err != nil {
	507	return false, err
	508	}
	509	var myOtherVal bool
	510	myOtherVal, err = evaluateOperator(fmt.Sprintf("%v", colToVal), "<=", checkStringType(myCoalVal))
	511	if err != nil {
	512	return false, err
	513	}
	514	return (myVal && myOtherVal), nil
	515	}
	516	myVal, err := evaluateOperator(record[columnNames[conversionColumn]], "<=", colFromVal)
	517	if err != nil {
	518	return false, err
	519	}
	520	var myOtherVal bool
	521	myOtherVal, err = evaluateOperator(fmt.Sprintf("%v", colToVal), "<=", checkStringType(record[columnNames[conversionColumn]]))
	522	if err != nil {
	523	return false, err
	524	}
	525	return (myVal && myOtherVal), nil
	526	}
	527
	528	// evalBetweenGreater is a function which evaluates the between given that the
	529	// TO is > than the FROM.
	530	func evalBetweenGreater(conversionColumn string, record []string, funcName string, columnNames map[string]int, colFromVal interface{}, colToVal interface{}, myCoalVal string) (bool, error) {
	531	if representsInt(conversionColumn) {
	532	myIndex, _ := strconv.Atoi(conversionColumn)
	533	myVal, err := evaluateOperator(record[myIndex-1], ">=", colFromVal)
	534	if err != nil {
	535	return false, err
	536	}
	537	var myOtherVal bool
	538	myOtherVal, err = evaluateOperator(fmt.Sprintf("%v", colToVal), ">=", checkStringType(record[myIndex-1]))
	539	if err != nil {
	540	return false, err
	541	}
	542	return (myVal && myOtherVal), nil
	543	}
	544	if myCoalVal != "" {
	545	myVal, err := evaluateOperator(myCoalVal, ">=", colFromVal)
	546	if err != nil {
	547	return false, err
	548	}
	549	var myOtherVal bool
	550	myOtherVal, err = evaluateOperator(fmt.Sprintf("%v", colToVal), ">=", checkStringType(myCoalVal))
	551	if err != nil {
	552	return false, err
	553	}
	554	return (myVal && myOtherVal), nil
	555	}
	556	myVal, err := evaluateOperator(record[columnNames[conversionColumn]], ">=", colFromVal)
	557	if err != nil {
	558	return false, err
	559	}
	560	var myOtherVal bool
	561	myOtherVal, err = evaluateOperator(fmt.Sprintf("%v", colToVal), ">=", checkStringType(record[columnNames[conversionColumn]]))
	562	if err != nil {
	563	return false, err
	564	}
	565	return (myVal && myOtherVal), nil
	566	}
	567
	568	// whereClauseNameErrs is a function which returns an error if there is a column
	569	// in the where clause which does not exist.
	570	func (reader *Input) whereClauseNameErrs(whereClause interface{}, alias string) error {
	571	var conversionColumn string
	572	switch expr := whereClause.(type) {
	573	// case for checking errors within a clause of the form "col_name is ..."
	574	case *sqlparser.IsExpr:
	575	switch myCol := expr.Expr.(type) {
	576	case *sqlparser.FuncExpr:
	577	if err := reader.evaluateFuncErr(myCol); err != nil {
	578	return err
	579	}
	580	case *sqlparser.ColName:
	581	conversionColumn = cleanCol(myCol.Name.CompliantName(), alias)
	582	}
	583	case *sqlparser.RangeCond:
	584	switch left := expr.Left.(type) {
	585	case *sqlparser.FuncExpr:
	586	if err := reader.evaluateFuncErr(left); err != nil {
	587	return err
	588	}
	589	case *sqlparser.ColName:
	590	conversionColumn = cleanCol(left.Name.CompliantName(), alias)
	591	}
	592	case *sqlparser.ComparisonExpr:
	593	switch left := expr.Left.(type) {
	594	case *sqlparser.FuncExpr:
	595	if err := reader.evaluateFuncErr(left); err != nil {
	596	return err
	597	}
	598	case *sqlparser.ColName:
	599	conversionColumn = cleanCol(left.Name.CompliantName(), alias)
	600	}
	601	case *sqlparser.AndExpr:
	602	switch left := expr.Left.(type) {
	603	case *sqlparser.ComparisonExpr:
	604	return reader.whereClauseNameErrs(left, alias)
	605	}
	606	switch right := expr.Right.(type) {
	607	case *sqlparser.ComparisonExpr:
	608	return reader.whereClauseNameErrs(right, alias)
	609	}
	610	case *sqlparser.OrExpr:
	611	switch left := expr.Left.(type) {
	612	case *sqlparser.ComparisonExpr:
	613	return reader.whereClauseNameErrs(left, alias)
	614	}
	615	switch right := expr.Right.(type) {
	616	case *sqlparser.ComparisonExpr:
	617	return reader.whereClauseNameErrs(right, alias)
	618	}
	619	}
	620	if conversionColumn != "" {
	621	return reader.colNameErrs([]string{conversionColumn})
	622	}
	623	return nil
	624	}
	625
	626	// qualityCheck ensures the row has enough separators.
	627	func qualityCheck(row string, amountOfSep int, sep string) string {
	628	for i := 0; i < amountOfSep; i++ {
	629	row = row + sep
	630	}
	631	return row
	632	}
	633
	634	// writeRow helps to write the row regardless of how many entries.
	635	func writeRow(myRow string, myEntry string, delimiter string, numOfReqCols int) string {
	636	if myEntry == "" && len(myRow) == 0 && numOfReqCols == 1 {
	637	return myEntry
	638	}
	639	if myEntry == "" && len(myRow) == 0 {
	640	return myEntry + delimiter
	641	}
	642	if len(myRow) == 1 && myRow[0] == ',' {
	643	return myRow + myEntry
	644	}
	645	if len(myRow) == 0 {
	646	return myEntry
	647	}
	648	return myRow + delimiter + myEntry
	649	}
	650
	651	// colNameErrs is a function which makes sure that the headers are requested are
	652	// present in the file otherwise it throws an error.
	653	func (reader *Input) colNameErrs(columnNames []string) error {
	654	for i := 0; i < len(columnNames); i++ {
	655	if columnNames[i] == "" {
	656	continue
	657	}
	658	if !representsInt(columnNames[i]) && !reader.options.HeaderOpt {
	659	return ErrInvalidColumnIndex
	660	}
	661	if representsInt(columnNames[i]) {
	662	tempInt, _ := strconv.Atoi(columnNames[i])
	663	if tempInt > len(reader.Header()) \|\| tempInt == 0 {
	664	return ErrInvalidColumnIndex
	665	}
	666	} else {
	667	if reader.options.HeaderOpt && !stringInSlice(columnNames[i], reader.Header()) {
	668	return ErrMissingHeaders
	669	}
	670	}
	671	}
	672	return nil
	673	}
	674
	675	// aggFuncToStr converts an array of floats into a properly formatted string.
	676	func (reader *Input) aggFuncToStr(myAggVals []float64) string {
	677	myRow := strconv.FormatFloat(myAggVals[0], 'f', 6, 64)
	678	for i := 1; i < len(myAggVals); i++ {
	679	aggregateval := strconv.FormatFloat(myAggVals[i], 'f', 6, 64)
	680	myRow = myRow + reader.options.OutputFieldDelimiter + aggregateval
	681	}
	682	return myRow
	683	}
	684
	685	// checkForDuplicates ensures we do not have an ambigious column name.
	686	func checkForDuplicates(columns []string, columnsMap map[string]int, hasDuplicates map[string]bool, lowercaseColumnsMap map[string]int) error {
	687	for i := 0; i < len(columns); i++ {
	688	columns[i] = strings.Replace(columns[i], " ", "_", len(columns[i]))
	689	if _, exist := columnsMap[columns[i]]; exist {
	690	return ErrAmbiguousFieldName
	691	}
	692	columnsMap[columns[i]] = i
	693	// This checks that if a key has already been put into the map, that we're
	694	// setting its appropriate value in has duplicates to be true.
	695	if _, exist := lowercaseColumnsMap[strings.ToLower(columns[i])]; exist {
	696	hasDuplicates[strings.ToLower(columns[i])] = true
	697	} else {
	698	lowercaseColumnsMap[strings.ToLower(columns[i])] = i
	699	}
	700	}
	701	return nil
	702	}
	703
	704	// evaluateParserType is a function that takes a SQL value and returns it as an
	705	// interface converted into the appropriate value.
	706	func evaluateParserType(col *sqlparser.SQLVal) (interface{}, error) {
	707	colDataType := col.Type
	708	var val interface{}
	709	switch colDataType {
	710	case 0:
	711	val = string(col.Val)
	712	case 1:
	713	intVersion, isInt := strconv.Atoi(string(col.Val))
	714	if isInt != nil {
	715	return nil, ErrIntegerOverflow
	716	}
	717	val = intVersion
	718	case 2:
	719	floatVersion, isFloat := strconv.ParseFloat(string(col.Val), 64)
	720	if isFloat != nil {
	721	return nil, ErrIntegerOverflow
	722	}
	723	val = floatVersion
	724	}
	725	return val, nil
	726	}
	727
	728	// parseErrs is the function which handles all the errors that could occur
	729	// through use of function arguments such as column names in NULLIF
	730	func (reader Input) parseErrs(columnNames []string, whereClause interface{}, alias string, myFuncs SelectFuncs) error {
	731	// Below code cleans up column names.
	732	reader.processColumnNames(columnNames, alias)
	733	if columnNames[0] != "*" {
	734	if err := reader.colNameErrs(columnNames); err != nil {
	735	return err
	736	}
	737	}
	738	// Below code ensures the whereClause has no errors.
	739	if whereClause != nil {
	740	tempClause := whereClause
	741	if err := reader.whereClauseNameErrs(tempClause, alias); err != nil {
	742	return err
	743	}
	744	}
	745	for i := 0; i < len(myFuncs.funcExpr); i++ {
	746	if myFuncs.funcExpr[i] == nil {
	747	continue
	748	}
	749	if err := reader.evaluateFuncErr(myFuncs.funcExpr[i]); err != nil {
	750	return err
	751	}
	752	}
	753	return nil
	754	}

pkg/s3select/input.go+381−0

@@ -0,0 +1,381 @@
	1	/*
	2	* Minio Cloud Storage, (C) 2018 Minio, Inc.
	3	*
	4	* Licensed under the Apache License, Version 2.0 (the "License");
	5	* you may not use this file except in compliance with the License.
	6	* You may obtain a copy of the License at
	7	*
	8	* http://www.apache.org/licenses/LICENSE-2.0
	9	*
	10	* Unless required by applicable law or agreed to in writing, software
	11	* distributed under the License is distributed on an "AS IS" BASIS,
	12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	* See the License for the specific language governing permissions and
	14	* limitations under the License.
	15	*/
	16
	17	package s3select
	18
	19	import (
	20	"bytes"
	21	"compress/bzip2"
	22	"encoding/csv"
	23	"encoding/xml"
	24	"io"
	25	"strconv"
	26	"strings"
	27	"time"
	28
	29	"net/http"
	30
	31	gzip "github.com/klauspost/pgzip"
	32	)
	33
	34	const (
	35	// progressTime is the time interval for which a progress message is sent.
	36	progressTime time.Duration = 60 * time.Second
	37	// continuationTime is the time interval for which a continuation message is
	38	// sent.
	39	continuationTime time.Duration = 5 * time.Second
	40	)
	41
	42	// progress represents a struct that represents the format for XML of the
	43	// progress messages
	44	type progress struct {
	45	BytesScanned int64 `xml:"BytesScanned"`
	46	BytesProcessed int64 `xml:"BytesProcessed"`
	47	BytesReturned int64 `xml:"BytesReturned"`
	48	Xmlns string `xml:"xmlns,attr"`
	49	}
	50
	51	// stats represents a struct that represents the format for XML of the stat
	52	// messages
	53	type stats struct {
	54	BytesScanned int64 `xml:"BytesScanned"`
	55	BytesProcessed int64 `xml:"BytesProcessed"`
	56	BytesReturned int64 `xml:"BytesReturned"`
	57	Xmlns string `xml:"xmlns,attr"`
	58	}
	59
	60	// StatInfo is a struct that represents the
	61	type statInfo struct {
	62	BytesScanned int64
	63	BytesReturned int64
	64	BytesProcessed int64
	65	}
	66
	67	// Input represents a record producing input from a formatted file or pipe.
	68	type Input struct {
	69	options *Options
	70	reader *csv.Reader
	71	firstRow []string
	72	header []string
	73	minOutputLength int
	74	stats *statInfo
	75	}
	76
	77	// Options options are passed to the underlying encoding/csv reader.
	78	type Options struct {
	79	// HasHeader when true, will treat the first row as a header row.
	80	HasHeader bool
	81
	82	// FieldDelimiter is the string that fields are delimited by.
	83	FieldDelimiter string
	84
	85	// Comments is the string the first character of a line of
	86	// text matches the comment character.
	87	Comments string
	88
	89	// Name of the table that is used for querying
	90	Name string
	91
	92	// ReadFrom is where the data will be read from.
	93	ReadFrom io.Reader
	94
	95	// If true then we need to add gzip or bzip reader.
	96	// to extract the csv.
	97	Compressed string
	98
	99	// SQL expression meant to be evaluated.
	100	Expression string
	101
	102	// What the outputted CSV will be delimited by .
	103	OutputFieldDelimiter string
	104
	105	// Size of incoming object
	106	StreamSize int64
	107
	108	// Whether Header is "USE" or another
	109	HeaderOpt bool
	110	}
	111
	112	// NewInput sets up a new Input, the first row is read when this is run.
	113	// If there is a problem with reading the first row, the error is returned.
	114	// Otherwise, the returned reader can be reliably consumed with ReadRecord()
	115	// until ReadRecord() returns nil.
	116	func NewInput(opts Options) (Input, error) {
	117	myReader := opts.ReadFrom
	118	var tempBytesScanned int64
	119	tempBytesScanned = 0
	120	if opts.Compressed == "GZIP" {
	121	tempBytesScanned = opts.StreamSize
	122	var err error
	123	if myReader, err = gzip.NewReader(opts.ReadFrom); err != nil {
	124	return nil, ErrTruncatedInput
	125	}
	126	} else if opts.Compressed == "BZIP2" {
	127	tempBytesScanned = opts.StreamSize
	128	myReader = bzip2.NewReader(opts.ReadFrom)
	129	}
	130
	131	progress := &statInfo{
	132	BytesScanned: tempBytesScanned,
	133	BytesProcessed: 0,
	134	BytesReturned: 0,
	135	}
	136	reader := &Input{
	137	options: opts,
	138	reader: csv.NewReader(myReader),
	139	stats: progress,
	140	}
	141	reader.firstRow = nil
	142
	143	reader.reader.FieldsPerRecord = -1
	144	if reader.options.FieldDelimiter != "" {
	145	reader.reader.Comma = rune(reader.options.FieldDelimiter[0])
	146	}
	147
	148	if reader.options.Comments != "" {
	149	reader.reader.Comment = rune(reader.options.Comments[0])
	150	}
	151
	152	// QuoteCharacter - " (defaulted currently)
	153	reader.reader.LazyQuotes = true
	154
	155	if err := reader.readHeader(); err != nil {
	156	return nil, err
	157	}
	158
	159	return reader, nil
	160	}
	161
	162	// ReadRecord reads a single record from the . Always returns successfully.
	163	// If the record is empty, an empty []string is returned.
	164	// Record expand to match the current row size, adding blank fields as needed.
	165	// Records never return less then the number of fields in the first row.
	166	// Returns nil on EOF
	167	// In the event of a parse error due to an invalid record, it is logged, and
	168	// an empty []string is returned with the number of fields in the first row,
	169	// as if the record were empty.
	170	//
	171	// In general, this is a very tolerant of problems reader.
	172	func (reader *Input) ReadRecord() []string {
	173	var row []string
	174	var fileErr error
	175
	176	if reader.firstRow != nil {
	177	row = reader.firstRow
	178	reader.firstRow = nil
	179	return row
	180	}
	181
	182	row, fileErr = reader.reader.Read()
	183	emptysToAppend := reader.minOutputLength - len(row)
	184	if fileErr == io.EOF \|\| fileErr == io.ErrClosedPipe {
	185	return nil
	186	} else if _, ok := fileErr.(*csv.ParseError); ok {
	187	emptysToAppend = reader.minOutputLength
	188	}
	189
	190	if emptysToAppend > 0 {
	191	for counter := 0; counter < emptysToAppend; counter++ {
	192	row = append(row, "")
	193	}
	194	}
	195
	196	return row
	197	}
	198
	199	// convertMySQL Replaces double quote escape for column names with backtick for
	200	// the MySQL parser
	201	func convertMySQL(random string) string {
	202	return strings.Replace(random, "\"", "`", len(random))
	203	}
	204
	205	// readHeader reads the header into the header variable if the header is present
	206	// as the first row of the csv
	207	func (reader *Input) readHeader() error {
	208	var readErr error
	209	if reader.options.HasHeader {
	210	reader.firstRow, readErr = reader.reader.Read()
	211	if readErr != nil {
	212	return ErrCSVParsingError
	213	}
	214	reader.header = reader.firstRow
	215	reader.firstRow = nil
	216	reader.minOutputLength = len(reader.header)
	217	} else {
	218	reader.firstRow, readErr = reader.reader.Read()
	219	reader.header = make([]string, len(reader.firstRow))
	220	for i := 0; i < reader.minOutputLength; i++ {
	221	reader.header[i] = strconv.Itoa(i)
	222	}
	223
	224	}
	225	return nil
	226	}
	227
	228	// createStatXML is the function which does the marshaling from the stat
	229	// structs into XML so that the progress and stat message can be sent
	230	func (reader *Input) createStatXML() (string, error) {
	231	if reader.options.Compressed == "NONE" {
	232	reader.stats.BytesProcessed = reader.options.StreamSize
	233	reader.stats.BytesScanned = reader.stats.BytesProcessed
	234	}
	235	statXML := stats{
	236	BytesScanned: reader.stats.BytesScanned,
	237	BytesProcessed: reader.stats.BytesProcessed,
	238	BytesReturned: reader.stats.BytesReturned,
	239	Xmlns: "",
	240	}
	241	out, err := xml.Marshal(statXML)
	242	if err != nil {
	243	return "", err
	244	}
	245	return xml.Header + string(out), nil
	246	}
	247
	248	// createProgressXML is the function which does the marshaling from the progress structs into XML so that the progress and stat message can be sent
	249	func (reader *Input) createProgressXML() (string, error) {
	250	if reader.options.HasHeader {
	251	reader.stats.BytesProcessed += processSize(reader.header)
	252	}
	253	if !(reader.options.Compressed != "NONE") {
	254	reader.stats.BytesScanned = reader.stats.BytesProcessed
	255	}
	256	progressXML := &progress{
	257	BytesScanned: reader.stats.BytesScanned,
	258	BytesProcessed: reader.stats.BytesProcessed,
	259	BytesReturned: reader.stats.BytesReturned,
	260	Xmlns: "",
	261	}
	262	out, err := xml.Marshal(progressXML)
	263	if err != nil {
	264	return "", err
	265	}
	266	return xml.Header + string(out), nil
	267	}
	268
	269	// Header returns the header of the reader. Either the first row if a header
	270	// set in the options, or c#, where # is the column number, starting with 0.
	271	func (reader *Input) Header() []string {
	272	return reader.header
	273	}
	274
	275	// Row is a Struct for keeping track of key aspects of a row.
	276	type Row struct {
	277	record string
	278	err error
	279	}
	280
	281	// Execute is the function where all the blocking occurs, It writes to the HTTP
	282	// response writer in a streaming fashion so that the client can actively use
	283	// the results before the query is finally finished executing. The
	284	func (reader *Input) Execute(writer io.Writer) error {
	285	myRow := make(chan *Row)
	286	curBuf := bytes.NewBuffer(make([]byte, 1000000))
	287	curBuf.Reset()
	288	progressTicker := time.NewTicker(progressTime)
	289	continuationTimer := time.NewTimer(continuationTime)
	290	defer progressTicker.Stop()
	291	defer continuationTimer.Stop()
	292	go reader.runSelectParser(convertMySQL(reader.options.Expression), myRow)
	293	for {
	294	select {
	295	case row, ok := <-myRow:
	296	if ok && row.err != nil {
	297	errorMessage := reader.writeErrorMessage(row.err, curBuf)
	298	_, err := errorMessage.WriteTo(writer)
	299	flusher, okFlush := writer.(http.Flusher)
	300	if okFlush {
	301	flusher.Flush()
	302	}
	303	if err != nil {
	304	return err
	305	}
	306	curBuf.Reset()
	307	close(myRow)
	308	return nil
	309	} else if ok {
	310	message := reader.writeRecordMessage(row.record, curBuf)
	311	_, err := message.WriteTo(writer)
	312	flusher, okFlush := writer.(http.Flusher)
	313	if okFlush {
	314	flusher.Flush()
	315	}
	316	if err != nil {
	317	return err
	318	}
	319	curBuf.Reset()
	320	reader.stats.BytesReturned += int64(len(row.record))
	321	if !continuationTimer.Stop() {
	322	<-continuationTimer.C
	323	}
	324	continuationTimer.Reset(continuationTime)
	325	} else if !ok {
	326	statPayload, err := reader.createStatXML()
	327	if err != nil {
	328	return err
	329	}
	330	statMessage := reader.writeStatMessage(statPayload, curBuf)
	331	_, err = statMessage.WriteTo(writer)
	332	flusher, ok := writer.(http.Flusher)
	333	if ok {
	334	flusher.Flush()
	335	}
	336	if err != nil {
	337	return err
	338	}
	339	curBuf.Reset()
	340	message := reader.writeEndMessage(curBuf)
	341	_, err = message.WriteTo(writer)
	342	flusher, ok = writer.(http.Flusher)
	343	if ok {
	344	flusher.Flush()
	345	}
	346	if err != nil {
	347	return err
	348	}
	349	return nil
	350	}
	351
	352	case <-progressTicker.C:
	353	progressPayload, err := reader.createProgressXML()
	354	if err != nil {
	355	return err
	356	}
	357	progressMessage := reader.writeProgressMessage(progressPayload, curBuf)
	358	_, err = progressMessage.WriteTo(writer)
	359	flusher, ok := writer.(http.Flusher)
	360	if ok {
	361	flusher.Flush()
	362	}
	363	if err != nil {
	364	return err
	365	}
	366	curBuf.Reset()
	367	case <-continuationTimer.C:
	368	message := reader.writeContinuationMessage(curBuf)
	369	_, err := message.WriteTo(writer)
	370	flusher, ok := writer.(http.Flusher)
	371	if ok {
	372	flusher.Flush()
	373	}
	374	if err != nil {
	375	return err
	376	}
	377	curBuf.Reset()
	378	continuationTimer.Reset(continuationTime)
	379	}
	380	}
	381	}

pkg/s3select/output.go+460−0

@@ -0,0 +1,460 @@
	1	/*
	2	* Minio Cloud Storage, (C) 2018 Minio, Inc.
	3	*
	4	* Licensed under the Apache License, Version 2.0 (the "License");
	5	* you may not use this file except in compliance with the License.
	6	* You may obtain a copy of the License at
	7	*
	8	* http://www.apache.org/licenses/LICENSE-2.0
	9	*
	10	* Unless required by applicable law or agreed to in writing, software
	11	* distributed under the License is distributed on an "AS IS" BASIS,
	12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	* See the License for the specific language governing permissions and
	14	* limitations under the License.
	15	*/
	16	// DO NOT EDIT THIS PACKAGE DIRECTLY: This follows the protocol defined by
	17	// AmazonS3 found at
	18	// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
	19	// Consult the Spec before making direct edits.
	20
	21	package s3select
	22
	23	import (
	24	"bytes"
	25	"encoding/binary"
	26	"hash/crc32"
	27	)
	28
	29	// Record Headers
	30	// -11 -event type - 7 - 7 "Records"
	31	// -13 -content-type -7 -24 "application/octet-stream"
	32	// -13 -message-type -7 5 "event"
	33	// This is predefined from AMZ protocol found here:
	34	// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
	35	var recordHeaders []byte
	36
	37	// End Headers
	38	// -13 -message-type -7 -5 "event"
	39	// -11 -:event-type -7 -3 "End"
	40	// This is predefined from AMZ protocol found here:
	41	// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
	42	var endHeaders []byte
	43
	44	// Continuation Headers
	45	// -13 -message-type -7 -5 "event"
	46	// -11 -:event-type -7 -4 "Cont"
	47	// This is predefined from AMZ protocol found here:
	48	// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
	49	var contHeaders []byte
	50
	51	// Stat Headers
	52	// -11 -event type - 7 - 5 "Stat" -20
	53	// -13 -content-type -7 -8 "text/xml" -25
	54	// -13 -message-type -7 -5 "event" -22
	55	// This is predefined from AMZ protocol found here:
	56	// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
	57	var statHeaders []byte
	58
	59	// Progress Headers
	60	// -11 -event type - 7 - 8 "Progress" -23
	61	// -13 -content-type -7 -8 "text/xml" -25
	62	// -13 -message-type -7 -5 "event" -22
	63	// This is predefined from AMZ protocol found here:
	64	// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
	65	var progressHeaders []byte
	66
	67	// The length of the nonvariable portion of the ErrHeaders
	68	// The below are the specifications of the header for a "error" event
	69	// -11 -error-code - 7 - DEFINED "DEFINED"
	70	// -14 -error-message -7 -DEFINED "DEFINED"
	71	// -13 -message-type -7 -5 "error"
	72	// This is predefined from AMZ protocol found here:
	73	// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
	74	var errHdrLen int
	75
	76	func init() {
	77	recordHeaders = writeRecordHeader()
	78	endHeaders = writeEndHeader()
	79	contHeaders = writeContHeader()
	80	statHeaders = writeStatHeader()
	81	progressHeaders = writeProgressHeader()
	82	errHdrLen = 55
	83
	84	}
	85
	86	// encodeString encodes a string in a []byte, lenBytes is the number of bytes
	87	// used to encode the length of the string.
	88	func encodeHeaderStringValue(s string) []byte {
	89	n := uint16(len(s))
	90	lenSlice := make([]byte, 2)
	91	binary.BigEndian.PutUint16(lenSlice[0:], n)
	92	return append(lenSlice, []byte(s)...)
	93	}
	94	func encodeHeaderStringName(s string) []byte {
	95	lenSlice := make([]byte, 1)
	96	lenSlice[0] = byte(len(s))
	97	return append(lenSlice, []byte(s)...)
	98	}
	99
	100	// encodeNumber encodes a number in a []byte, lenBytes is the number of bytes
	101	// used to encode the length of the string.
	102	func encodeNumber(n byte, lenBytes int) []byte {
	103	lenSlice := make([]byte, lenBytes)
	104	lenSlice[0] = n
	105	return lenSlice
	106	}
	107
	108	// writePayloadSize writes the 4byte payload size portion of the protocol.
	109	func writePayloadSize(payloadSize int, headerLength int) []byte {
	110	totalByteLen := make([]byte, 4)
	111	totalMsgLen := uint32(payloadSize + headerLength + 16)
	112	binary.BigEndian.PutUint32(totalByteLen, totalMsgLen)
	113	return totalByteLen
	114	}
	115
	116	// writeHeaderSize writes the 4byte header size portion of the protocol.
	117	func writeHeaderSize(headerLength int) []byte {
	118	totalHeaderLen := make([]byte, 4)
	119	totalLen := uint32(headerLength)
	120	binary.BigEndian.PutUint32(totalHeaderLen, totalLen)
	121	return totalHeaderLen
	122	}
	123
	124	// writeCRC writes the CRC for both the prelude and and the end of the protocol.
	125	func writeCRC(myBuffer []byte) []byte {
	126	// Calculate the CRC here:
	127	myCRC := make([]byte, 4)
	128	cksum := crc32.ChecksumIEEE(myBuffer)
	129	binary.BigEndian.PutUint32(myCRC, cksum)
	130	return myCRC
	131	}
	132
	133	// writePayload writes the Payload for those protocols which the Payload is
	134	// necessary.
	135	func writePayload(myPayload string) []byte {
	136	convertedPayload := []byte(myPayload)
	137	payloadStore := make([]byte, len(convertedPayload))
	138	copy(payloadStore[0:], myPayload)
	139	return payloadStore
	140	}
	141
	142	// writeRecordHeader is a function which writes the headers for the continuation
	143	// Message
	144	func writeRecordHeader() []byte {
	145	// This is predefined from AMZ protocol found here:
	146	// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
	147	var currentMessage = &bytes.Buffer{}
	148	// 11 -event type - 7 - 7 "Records"
	149	// header name
	150	currentMessage.Write(encodeHeaderStringName(":event-type"))
	151	// header type
	152	currentMessage.Write(encodeNumber(7, 1))
	153	// header value and header value length
	154	currentMessage.Write(encodeHeaderStringValue("Records"))
	155	// Creation of the Header for Content-Type // 13 -content-type -7 -24
	156	// "application/octet-stream"
	157	// header name
	158	currentMessage.Write(encodeHeaderStringName(":content-type"))
	159	// header type
	160	currentMessage.Write(encodeNumber(7, 1))
	161	// header value and header value length
	162	currentMessage.Write(encodeHeaderStringValue("application/octet-stream"))
	163	// Creation of the Header for message-type 13 -message-type -7 5 "event"
	164	// header name
	165	currentMessage.Write(encodeHeaderStringName(":message-type"))
	166	// header type
	167	currentMessage.Write(encodeNumber(7, 1))
	168	// header value and header value length
	169	currentMessage.Write(encodeHeaderStringValue("event"))
	170	return currentMessage.Bytes()
	171	}
	172
	173	// writeEndHeader is a function which writes the headers for the continuation
	174	// Message
	175	func writeEndHeader() []byte {
	176	// This is predefined from AMZ protocol found here:
	177	// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
	178	var currentMessage = &bytes.Buffer{}
	179	// header name
	180	currentMessage.Write(encodeHeaderStringName(":event-type"))
	181	// header type
	182	currentMessage.Write(encodeNumber(7, 1))
	183	// header value and header value length
	184	currentMessage.Write(encodeHeaderStringValue("End"))
	185
	186	// Creation of the Header for message-type 13 -message-type -7 5 "event"
	187	// header name
	188	currentMessage.Write(encodeHeaderStringName(":message-type"))
	189	// header type
	190	currentMessage.Write(encodeNumber(7, 1))
	191	// header value and header value length
	192	currentMessage.Write(encodeHeaderStringValue("event"))
	193	return currentMessage.Bytes()
	194	}
	195
	196	// writeContHeader is a function which writes the headers for the continuation
	197	// Message
	198	func writeContHeader() []byte {
	199	// This is predefined from AMZ protocol found here:
	200	// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
	201	var currentMessage = &bytes.Buffer{}
	202	// header name
	203	currentMessage.Write(encodeHeaderStringName(":event-type"))
	204	// header type
	205	currentMessage.Write(encodeNumber(7, 1))
	206	// header value and header value length
	207	currentMessage.Write(encodeHeaderStringValue("Cont"))
	208
	209	// Creation of the Header for message-type 13 -message-type -7 5 "event"
	210	// header name
	211	currentMessage.Write(encodeHeaderStringName(":message-type"))
	212	// header type
	213	currentMessage.Write(encodeNumber(7, 1))
	214	// header value and header value length
	215	currentMessage.Write(encodeHeaderStringValue("event"))
	216	return currentMessage.Bytes()
	217
	218	}
	219
	220	// writeStatHeader is a function which writes the headers for the Stat
	221	// Message
	222	func writeStatHeader() []byte {
	223	// This is predefined from AMZ protocol found here:
	224	// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
	225	var currentMessage = &bytes.Buffer{}
	226	// header name
	227	currentMessage.Write(encodeHeaderStringName(":event-type"))
	228	// header type
	229	currentMessage.Write(encodeNumber(7, 1))
	230	// header value and header value length
	231	currentMessage.Write(encodeHeaderStringValue("Stats"))
	232	// Creation of the Header for Content-Type // 13 -content-type -7 -8
	233	// "text/xml"
	234	// header name
	235	currentMessage.Write(encodeHeaderStringName(":content-type"))
	236	// header type
	237	currentMessage.Write(encodeNumber(7, 1))
	238	// header value and header value length
	239	currentMessage.Write(encodeHeaderStringValue("text/xml"))
	240
	241	// Creation of the Header for message-type 13 -message-type -7 5 "event"
	242	currentMessage.Write(encodeHeaderStringName(":message-type"))
	243	// header type
	244	currentMessage.Write(encodeNumber(7, 1))
	245	// header value and header value length
	246	currentMessage.Write(encodeHeaderStringValue("event"))
	247	return currentMessage.Bytes()
	248
	249	}
	250
	251	// writeProgressHeader is a function which writes the headers for the Progress
	252	// Message
	253	func writeProgressHeader() []byte {
	254	// This is predefined from AMZ protocol found here:
	255	// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
	256	var currentMessage = &bytes.Buffer{}
	257	// header name
	258	currentMessage.Write(encodeHeaderStringName(":event-type"))
	259	// header type
	260	currentMessage.Write(encodeNumber(7, 1))
	261	// header value and header value length
	262	currentMessage.Write(encodeHeaderStringValue("Progress"))
	263	// Creation of the Header for Content-Type // 13 -content-type -7 -8
	264	// "text/xml"
	265	// header name
	266	currentMessage.Write(encodeHeaderStringName(":content-type"))
	267	// header type
	268	currentMessage.Write(encodeNumber(7, 1))
	269	// header value and header value length
	270	currentMessage.Write(encodeHeaderStringValue("text/xml"))
	271
	272	// Creation of the Header for message-type 13 -message-type -7 5 "event"
	273	// header name
	274	currentMessage.Write(encodeHeaderStringName(":message-type"))
	275	// header type
	276	currentMessage.Write(encodeNumber(7, 1))
	277	// header value and header value length
	278	currentMessage.Write(encodeHeaderStringValue("event"))
	279	return currentMessage.Bytes()
	280
	281	}
	282
	283	// writeRecordMessage is the function which constructs the binary message for a
	284	// record message to be sent.
	285	func (csvOutput Input) writeRecordMessage(payload string, currentMessage bytes.Buffer) *bytes.Buffer {
	286	// The below are the specifications of the header for a "record" event
	287	// 11 -event type - 7 - 7 "Records"
	288	// 13 -content-type -7 -24 "application/octet-stream"
	289	// 13 -message-type -7 5 "event"
	290	// This is predefined from AMZ protocol found here:
	291	// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
	292	headerLen := len(recordHeaders)
	293	// Writes the total size of the message.
	294	currentMessage.Write(writePayloadSize(len(payload), headerLen))
	295	// Writes the total size of the header.
	296	currentMessage.Write(writeHeaderSize(headerLen))
	297	// Writes the CRC of the Prelude
	298	currentMessage.Write(writeCRC(currentMessage.Bytes()))
	299	currentMessage.Write(recordHeaders)
	300
	301	// This part is where the payload is written, this will be only one row, since
	302	// we're sending one message at a types
	303	currentMessage.Write(writePayload(payload))
	304
	305	// Now we do a CRC check on the entire messages
	306	currentMessage.Write(writeCRC(currentMessage.Bytes()))
	307	return currentMessage
	308
	309	}
	310
	311	// writeContinuationMessage is the function which constructs the binary message
	312	// for a continuation message to be sent.
	313	func (csvOutput Input) writeContinuationMessage(currentMessage bytes.Buffer) *bytes.Buffer {
	314	// 11 -event type - 7 - 4 "Cont"
	315	// 13 -message-type -7 5 "event"
	316	// This is predefined from AMZ protocol found here:
	317	// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
	318	headerLen := len(contHeaders)
	319	currentMessage.Write(writePayloadSize(0, headerLen))
	320
	321	currentMessage.Write(writeHeaderSize(headerLen))
	322
	323	// Calculate the Prelude CRC here:
	324	currentMessage.Write(writeCRC(currentMessage.Bytes()))
	325
	326	currentMessage.Write(contHeaders)
	327
	328	//Now we do a CRC check on the entire messages
	329	currentMessage.Write(writeCRC(currentMessage.Bytes()))
	330	return currentMessage
	331
	332	}
	333
	334	// writeEndMessage is the function which constructs the binary message
	335	// for a end message to be sent.
	336	func (csvOutput Input) writeEndMessage(currentMessage bytes.Buffer) *bytes.Buffer {
	337	// 11 -event type - 7 - 3 "End"
	338	// 13 -message-type -7 5 "event"
	339	// This is predefined from AMZ protocol found here:
	340	// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
	341	headerLen := len(endHeaders)
	342	currentMessage.Write(writePayloadSize(0, headerLen))
	343
	344	currentMessage.Write(writeHeaderSize(headerLen))
	345
	346	//Calculate the Prelude CRC here:
	347	currentMessage.Write(writeCRC(currentMessage.Bytes()))
	348
	349	currentMessage.Write(endHeaders)
	350
	351	// Now we do a CRC check on the entire messages
	352	currentMessage.Write(writeCRC(currentMessage.Bytes()))
	353	return currentMessage
	354
	355	}
	356
	357	// writeStateMessage is the function which constructs the binary message for a
	358	// state message to be sent.
	359	func (csvOutput Input) writeStatMessage(payload string, currentMessage bytes.Buffer) *bytes.Buffer {
	360	// 11 -event type - 7 - 5 "Stat" 20
	361	// 13 -content-type -7 -8 "text/xml" 25
	362	// 13 -message-type -7 5 "event" 22
	363	// This is predefined from AMZ protocol found here:
	364	// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
	365	headerLen := len(statHeaders)
	366
	367	currentMessage.Write(writePayloadSize(len(payload), headerLen))
	368
	369	currentMessage.Write(writeHeaderSize(headerLen))
	370
	371	currentMessage.Write(writeCRC(currentMessage.Bytes()))
	372
	373	currentMessage.Write(statHeaders)
	374
	375	// This part is where the payload is written, this will be only one row, since
	376	// we're sending one message at a types
	377	currentMessage.Write(writePayload(payload))
	378
	379	// Now we do a CRC check on the entire messages
	380	currentMessage.Write(writeCRC(currentMessage.Bytes()))
	381	return currentMessage
	382
	383	}
	384
	385	// writeProgressMessage is the function which constructs the binary message for
	386	// a progress message to be sent.
	387	func (csvOutput Input) writeProgressMessage(payload string, currentMessage bytes.Buffer) *bytes.Buffer {
	388	// The below are the specifications of the header for a "Progress" event
	389	// 11 -event type - 7 - 8 "Progress" 23
	390	// 13 -content-type -7 -8 "text/xml" 25
	391	// 13 -message-type -7 5 "event" 22
	392	// This is predefined from AMZ protocol found here:
	393	// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
	394	headerLen := len(progressHeaders)
	395
	396	currentMessage.Write(writePayloadSize(len(payload), headerLen))
	397
	398	currentMessage.Write(writeHeaderSize(headerLen))
	399
	400	currentMessage.Write(writeCRC(currentMessage.Bytes()))
	401
	402	currentMessage.Write(progressHeaders)
	403
	404	// This part is where the payload is written, this will be only one row, since
	405	// we're sending one message at a types
	406	currentMessage.Write(writePayload(payload))
	407
	408	// Now we do a CRC check on the entire messages
	409	currentMessage.Write(writeCRC(currentMessage.Bytes()))
	410	return currentMessage
	411
	412	}
	413
	414	// writeErrorMessage is the function which constructs the binary message for a
	415	// error message to be sent.
	416	func (csvOutput Input) writeErrorMessage(errorMessage error, currentMessage bytes.Buffer) *bytes.Buffer {
	417
	418	// The below are the specifications of the header for a "error" event
	419	// 11 -error-code - 7 - DEFINED "DEFINED"
	420	// 14 -error-message -7 -DEFINED "DEFINED"
	421	// 13 -message-type -7 5 "error"
	422	// This is predefined from AMZ protocol found here:
	423	// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
	424	sizeOfErrorCode := len(errorCodeResponse[errorMessage])
	425	sizeOfErrorMessage := len(errorMessage.Error())
	426	headerLen := errHdrLen + sizeOfErrorCode + sizeOfErrorMessage
	427
	428	currentMessage.Write(writePayloadSize(0, headerLen))
	429
	430	currentMessage.Write(writeHeaderSize(headerLen))
	431
	432	currentMessage.Write(writeCRC(currentMessage.Bytes()))
	433	// header name
	434	currentMessage.Write(encodeHeaderStringName(":error-code"))
	435	// header type
	436	currentMessage.Write(encodeNumber(7, 1))
	437	// header value and header value length
	438	currentMessage.Write(encodeHeaderStringValue(errorCodeResponse[errorMessage]))
	439
	440	// 14 -error-message -7 -DEFINED "DEFINED"
	441
	442	// header name
	443	currentMessage.Write(encodeHeaderStringName(":error-message"))
	444	// header type
	445	currentMessage.Write(encodeNumber(7, 1))
	446	// header value and header value length
	447	currentMessage.Write(encodeHeaderStringValue(errorMessage.Error()))
	448	// Creation of the Header for message-type 13 -message-type -7 5 "error"
	449	// header name
	450	currentMessage.Write(encodeHeaderStringName(":message-type"))
	451	// header type
	452	currentMessage.Write(encodeNumber(7, 1))
	453	// header value and header value length
	454	currentMessage.Write(encodeHeaderStringValue("error"))
	455
	456	// Now we do a CRC check on the entire messages
	457	currentMessage.Write(writeCRC(currentMessage.Bytes()))
	458	return currentMessage
	459
	460	}

pkg/s3select/select.go+415−0

@@ -0,0 +1,415 @@
	1	/*
	2	* Minio Cloud Storage, (C) 2018 Minio, Inc.
	3	*
	4	* Licensed under the Apache License, Version 2.0 (the "License");
	5	* you may not use this file except in compliance with the License.
	6	* You may obtain a copy of the License at
	7	*
	8	* http://www.apache.org/licenses/LICENSE-2.0
	9	*
	10	* Unless required by applicable law or agreed to in writing, software
	11	* distributed under the License is distributed on an "AS IS" BASIS,
	12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	* See the License for the specific language governing permissions and
	14	* limitations under the License.
	15	*/
	16
	17	package s3select
	18
	19	import (
	20	"math"
	21	"strconv"
	22	"strings"
	23
	24	"github.com/xwb1989/sqlparser"
	25	)
	26
	27	// SelectFuncs contains the relevant values from the parser for S3 Select
	28	// Functions
	29	type SelectFuncs struct {
	30	funcExpr []*sqlparser.FuncExpr
	31	index []int
	32	}
	33
	34	// RunSqlParser allows us to easily bundle all the functions from above and run
	35	// them in the appropriate order.
	36	func (reader Input) runSelectParser(selectExpression string, myRow chan Row) {
	37	reqCols, alias, myLimit, whereClause, aggFunctionNames, myFuncs, myErr := reader.ParseSelect(selectExpression)
	38	if myErr != nil {
	39	rowStruct := &Row{
	40	err: myErr,
	41	}
	42	myRow <- rowStruct
	43	return
	44	}
	45	reader.processSelectReq(reqCols, alias, whereClause, myLimit, aggFunctionNames, myRow, myFuncs)
	46	}
	47
	48	// ParseSelect parses the SELECT expression, and effectively tokenizes it into
	49	// its separate parts. It returns the requested column names,alias,limit of
	50	// records, and the where clause.
	51	func (reader Input) ParseSelect(sqlInput string) ([]string, string, int, interface{}, []string, SelectFuncs, error) {
	52	// return columnNames, alias, limitOfRecords, whereclause,coalStore, nil
	53
	54	stmt, err := sqlparser.Parse(sqlInput)
	55	var whereClause interface{}
	56	var alias string
	57	var limit int
	58	myFuncs := &SelectFuncs{}
	59	// TODO Maybe can parse their errors a bit to return some more of the s3 errors
	60	if err != nil {
	61	return nil, "", 0, nil, nil, nil, ErrLexerInvalidChar
	62	}
	63	switch stmt := stmt.(type) {
	64	case *sqlparser.Select:
	65	// evaluates the where clause
	66	functionNames := make([]string, len(stmt.SelectExprs))
	67	columnNames := make([]string, len(stmt.SelectExprs))
	68
	69	if stmt.Where != nil {
	70	switch expr := stmt.Where.Expr.(type) {
	71	default:
	72	whereClause = expr
	73	case *sqlparser.ComparisonExpr:
	74	whereClause = expr
	75	}
	76	}
	77	if stmt.SelectExprs != nil {
	78	for i := 0; i < len(stmt.SelectExprs); i++ {
	79	switch expr := stmt.SelectExprs[i].(type) {
	80	case *sqlparser.StarExpr:
	81	columnNames[0] = "*"
	82	case *sqlparser.AliasedExpr:
	83	switch smallerexpr := expr.Expr.(type) {
	84	case *sqlparser.FuncExpr:
	85	if smallerexpr.IsAggregate() {
	86	functionNames[i] = smallerexpr.Name.CompliantName()
	87	// Will return function name
	88	// Case to deal with if we have functions and not an asterix
	89	switch tempagg := smallerexpr.Exprs[0].(type) {
	90	case *sqlparser.StarExpr:
	91	columnNames[0] = "*"
	92	if smallerexpr.Name.CompliantName() != "count" {
	93	return nil, "", 0, nil, nil, nil, ErrParseUnsupportedCallWithStar
	94	}
	95	case *sqlparser.AliasedExpr:
	96	switch col := tempagg.Expr.(type) {
	97	case *sqlparser.BinaryExpr:
	98	return nil, "", 0, nil, nil, nil, ErrParseNonUnaryAgregateFunctionCall
	99	case *sqlparser.ColName:
	100	columnNames[i] = col.Name.CompliantName()
	101	}
	102	}
	103	// Case to deal with if COALESCE was used..
	104	} else if supportedFunc(smallerexpr.Name.CompliantName()) {
	105	if myFuncs.funcExpr == nil {
	106	myFuncs.funcExpr = make([]*sqlparser.FuncExpr, len(stmt.SelectExprs))
	107	myFuncs.index = make([]int, len(stmt.SelectExprs))
	108	}
	109	myFuncs.funcExpr[i] = smallerexpr
	110	myFuncs.index[i] = i
	111	} else {
	112	return nil, "", 0, nil, nil, nil, ErrUnsupportedSQLOperation
	113	}
	114	case *sqlparser.ColName:
	115	columnNames[i] = smallerexpr.Name.CompliantName()
	116	}
	117	}
	118	}
	119	}
	120
	121	// This code retrieves the alias and makes sure it is set to the correct
	122	// value, if not it sets it to the tablename
	123	if (stmt.From) != nil {
	124	for i := 0; i < len(stmt.From); i++ {
	125	switch smallerexpr := stmt.From[i].(type) {
	126	case *sqlparser.JoinTableExpr:
	127	return nil, "", 0, nil, nil, nil, ErrParseMalformedJoin
	128	case *sqlparser.AliasedTableExpr:
	129	alias = smallerexpr.As.CompliantName()
	130	if alias == "" {
	131	alias = sqlparser.GetTableName(smallerexpr.Expr).CompliantName()
	132	}
	133	}
	134	}
	135	}
	136	if stmt.Limit != nil {
	137	switch expr := stmt.Limit.Rowcount.(type) {
	138	case *sqlparser.SQLVal:
	139	// The Value of how many rows we're going to limit by
	140	limit, _ = strconv.Atoi(string(expr.Val[:]))
	141	}
	142	}
	143	if stmt.GroupBy != nil {
	144	return nil, "", 0, nil, nil, nil, ErrParseUnsupportedLiteralsGroupBy
	145	}
	146	if stmt.OrderBy != nil {
	147	return nil, "", 0, nil, nil, nil, ErrParseUnsupportedToken
	148	}
	149	if err := reader.parseErrs(columnNames, whereClause, alias, myFuncs); err != nil {
	150	return nil, "", 0, nil, nil, nil, err
	151	}
	152	return columnNames, alias, limit, whereClause, functionNames, myFuncs, nil
	153	}
	154	return nil, "", 0, nil, nil, nil, nil
	155	}
	156
	157	// This is the main function, It goes row by row and for records which validate
	158	// the where clause it currently prints the appropriate row given the requested
	159	// columns.
	160	func (reader Input) processSelectReq(reqColNames []string, alias string, whereClause interface{}, limitOfRecords int, functionNames []string, myRow chan Row, myFunc *SelectFuncs) {
	161	counter := -1
	162	filtrCount := 0
	163	functionFlag := false
	164	// My values is used to store our aggregation values if we need to store them.
	165	myAggVals := make([]float64, len(reqColNames))
	166	var columns []string
	167	// LowercasecolumnsMap is used in accordance with hasDuplicates so that we can
	168	// raise the error "Ambigious" if a case insensitive column is provided and we
	169	// have multiple matches.
	170	lowercaseColumnsMap := make(map[string]int)
	171	hasDuplicates := make(map[string]bool)
	172	// ColumnsMap stores our columns and their index.
	173	columnsMap := make(map[string]int)
	174	if limitOfRecords == 0 {
	175	limitOfRecords = math.MaxInt64
	176	}
	177
	178	for {
	179	record := reader.ReadRecord()
	180	reader.stats.BytesProcessed += processSize(record)
	181	if record == nil {
	182	if functionFlag {
	183	rowStruct := &Row{
	184	record: reader.aggFuncToStr(myAggVals) + "\n",
	185	}
	186	myRow <- rowStruct
	187	}
	188	close(myRow)
	189	return
	190	}
	191	if counter == -1 && reader.options.HeaderOpt && len(reader.header) > 0 {
	192	columns = reader.Header()
	193	myErr := checkForDuplicates(columns, columnsMap, hasDuplicates, lowercaseColumnsMap)
	194	if myErr != nil {
	195	rowStruct := &Row{
	196	err: myErr,
	197	}
	198	myRow <- rowStruct
	199	return
	200	}
	201	} else if counter == -1 && len(reader.header) > 0 {
	202	columns = reader.Header()
	203	}
	204	// When we have reached our limit, on what the user specified as the number
	205	// of rows they wanted, we terminate our interpreter.
	206	if filtrCount == limitOfRecords && limitOfRecords != 0 {
	207	close(myRow)
	208	return
	209	}
	210	// The call to the where function clause,ensures that the rows we print match our where clause.
	211	condition, myErr := matchesMyWhereClause(record, columnsMap, alias, whereClause)
	212	if myErr != nil {
	213	rowStruct := &Row{
	214	err: myErr,
	215	}
	216	myRow <- rowStruct
	217	return
	218	}
	219	if condition {
	220	// if its an asterix we just print everything in the row
	221	if reqColNames[0] == "*" && functionNames[0] == "" {
	222	rowStruct := &Row{
	223	record: reader.printAsterix(record) + "\n",
	224	}
	225	myRow <- rowStruct
	226	} else if alias != "" {
	227	// This is for dealing with the case of if we have to deal with a
	228	// request for a column with an index e.g A_1.
	229	if representsInt(reqColNames[0]) {
	230	// This checks whether any aggregation function was called as now we
	231	// no longer will go through printing each row, and only print at the
	232	// end
	233	if len(functionNames) > 0 && functionNames[0] != "" {
	234	functionFlag = true
	235	aggregationFunctions(counter, filtrCount, myAggVals, columnsMap, reqColNames, functionNames, record)
	236	} else {
	237	// The code below finds the appropriate columns of the row given the
	238	// indicies provided in the SQL request and utilizes the map to
	239	// retrieve the correct part of the row.
	240	myQueryRow, myErr := reader.processColNameIndex(record, reqColNames, columns)
	241	if myErr != nil {
	242	rowStruct := &Row{
	243	err: myErr,
	244	}
	245	myRow <- rowStruct
	246	return
	247	}
	248	rowStruct := &Row{
	249	record: myQueryRow + "\n",
	250	}
	251	myRow <- rowStruct
	252	}
	253	} else {
	254	// This code does aggregation if we were provided column names in the
	255	// form of acutal names rather an indices.
	256	if len(functionNames) > 0 && functionNames[0] != "" {
	257	functionFlag = true
	258	aggregationFunctions(counter, filtrCount, myAggVals, columnsMap, reqColNames, functionNames, record)
	259	} else {
	260	// This code prints the appropriate part of the row given the filter
	261	// and select request, if the select request was based on column
	262	// names rather than indices.
	263	myQueryRow, myErr := reader.processColNameLiteral(record, reqColNames, columns, columnsMap, myFunc)
	264	if myErr != nil {
	265	rowStruct := &Row{
	266	err: myErr,
	267	}
	268	myRow <- rowStruct
	269	return
	270	}
	271	rowStruct := &Row{
	272	record: myQueryRow + "\n",
	273	}
	274	myRow <- rowStruct
	275	}
	276	}
	277	}
	278	filtrCount++
	279	}
	280	counter++
	281	}
	282	}
	283
	284	// printAsterix helps to print out the entire row if an asterix is used.
	285	func (reader *Input) printAsterix(record []string) string {
	286	myRow := record[0]
	287	for i := 1; i < len(record); i++ {
	288	myRow = myRow + reader.options.OutputFieldDelimiter + record[i]
	289	}
	290	return myRow
	291	}
	292
	293	// processColumnNames is a function which allows for cleaning of column names.
	294	func (reader *Input) processColumnNames(reqColNames []string, alias string) error {
	295	for i := 0; i < len(reqColNames); i++ {
	296	// The code below basically cleans the column name of its alias and other
	297	// syntax, so that we can extract its pure name.
	298	reqColNames[i] = cleanCol(reqColNames[i], alias)
	299	}
	300	return nil
	301	}
	302
	303	// processColNameIndex is the function which creates the row for an index based
	304	// query.
	305	func (reader *Input) processColNameIndex(record []string, reqColNames []string, columns []string) (string, error) {
	306	myRow := ""
	307	for i := 0; i < len(reqColNames); i++ {
	308	// COALESCE AND NULLIF do not support index based access.
	309	if reqColNames[0] == "0" {
	310	return "", ErrInvalidColumnIndex
	311	}
	312	// Subtract 1 because AWS Indexing is not 0 based, it starts at 1.
	313	mytempindex, err := strconv.Atoi(reqColNames[i])
	314	mytempindex = mytempindex - 1
	315	if mytempindex > len(columns) {
	316	return "", ErrInvalidColumnIndex
	317	}
	318	myRow = writeRow(myRow, record[mytempindex], reader.options.OutputFieldDelimiter, len(reqColNames))
	319	if err != nil {
	320	return "", ErrMissingHeaders
	321	}
	322	}
	323	if len(myRow) > 1000000 {
	324	return "", ErrOverMaxRecordSize
	325	}
	326	if strings.Count(myRow, reader.options.OutputFieldDelimiter) != len(reqColNames)-1 {
	327	myRow = qualityCheck(myRow, len(reqColNames)-1-strings.Count(myRow, reader.options.OutputFieldDelimiter), reader.options.OutputFieldDelimiter)
	328	}
	329	return myRow, nil
	330	}
	331
	332	// processColNameLiteral is the function which creates the row for an name based
	333	// query.
	334	func (reader Input) processColNameLiteral(record []string, reqColNames []string, columns []string, columnsMap map[string]int, myFunc SelectFuncs) (string, error) {
	335	myRow := ""
	336	for i := 0; i < len(reqColNames); i++ {
	337	// this is the case to deal with COALESCE.
	338	if reqColNames[i] == "" && isValidFunc(myFunc.index, i) {
	339	myVal := evaluateFuncExpr(myFunc.funcExpr[i], "", record, columnsMap)
	340	myRow = writeRow(myRow, myVal, reader.options.OutputFieldDelimiter, len(reqColNames))
	341	continue
	342	}
	343	myTempIndex, notFound := columnsMap[trimQuotes(reqColNames[i])]
	344	if !notFound {
	345	return "", ErrMissingHeaders
	346	}
	347	myRow = writeRow(myRow, record[myTempIndex], reader.options.OutputFieldDelimiter, len(reqColNames))
	348	}
	349	if len(myRow) > 1000000 {
	350	return "", ErrOverMaxRecordSize
	351	}
	352	if strings.Count(myRow, reader.options.OutputFieldDelimiter) != len(reqColNames)-1 {
	353	myRow = qualityCheck(myRow, len(reqColNames)-1-strings.Count(myRow, reader.options.OutputFieldDelimiter), reader.options.OutputFieldDelimiter)
	354	}
	355	return myRow, nil
	356
	357	}
	358
	359	// aggregationFunctions is a function which performs the actual aggregation
	360	// methods on the given row, it uses an array defined the the main parsing
	361	// function to keep track of values.
	362	func aggregationFunctions(counter int, filtrCount int, myAggVals []float64, columnsMap map[string]int, storeReqCols []string, storeFunctions []string, record []string) error {
	363	for i := 0; i < len(storeFunctions); i++ {
	364	if storeFunctions[i] == "" {
	365	i++
	366	} else if storeFunctions[i] == "count" {
	367	myAggVals[i]++
	368	} else {
	369	// If column names are provided as an index it'll use this if statement instead of the else/
	370	var convAggFloat float64
	371	if representsInt(storeReqCols[i]) {
	372	myIndex, _ := strconv.Atoi(storeReqCols[i])
	373	convAggFloat, _ = strconv.ParseFloat(record[myIndex], 64)
	374
	375	} else {
	376	// case that the columns are in the form of named columns rather than indices.
	377	convAggFloat, _ = strconv.ParseFloat(record[columnsMap[trimQuotes(storeReqCols[i])]], 64)
	378
	379	}
	380	// This if statement is for calculating the min.
	381	if storeFunctions[i] == "min" {
	382	if counter == -1 {
	383	myAggVals[i] = math.MaxFloat64
	384	}
	385	if convAggFloat < myAggVals[i] {
	386	myAggVals[i] = convAggFloat
	387	}
	388
	389	} else if storeFunctions[i] == "max" {
	390	// This if statement is for calculating the max.
	391	if counter == -1 {
	392	myAggVals[i] = math.SmallestNonzeroFloat64
	393	}
	394	if convAggFloat > myAggVals[i] {
	395	myAggVals[i] = convAggFloat
	396	}
	397
	398	} else if storeFunctions[i] == "sum" {
	399	// This if statement is for calculating the sum.
	400	myAggVals[i] += convAggFloat
	401
	402	} else if storeFunctions[i] == "avg" {
	403	// This if statement is for calculating the average.
	404	if filtrCount == 0 {
	405	myAggVals[i] = convAggFloat
	406	} else {
	407	myAggVals[i] = (convAggFloat + (myAggVals[i] * float64(filtrCount))) / float64((filtrCount + 1))
	408	}
	409	} else {
	410	return ErrParseNonUnaryAgregateFunctionCall
	411	}
	412	}
	413	}
	414	return nil
	415	}

pkg/s3select/select_test.go+1070−0

@@ -0,0 +1,1070 @@
	1	/*
	2	* Minio Cloud Storage, (C) 2018 Minio, Inc.
	3	*
	4	* Licensed under the Apache License, Version 2.0 (the "License");
	5	* you may not use this file except in compliance with the License.
	6	* You may obtain a copy of the License at
	7	*
	8	* http://www.apache.org/licenses/LICENSE-2.0
	9	*
	10	* Unless required by applicable law or agreed to in writing, software
	11	* distributed under the License is distributed on an "AS IS" BASIS,
	12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	* See the License for the specific language governing permissions and
	14	* limitations under the License.
	15	*/
	16
	17	package s3select
	18
	19	import (
	20	"bytes"
	21	"fmt"
	22	"reflect"
	23	"testing"
	24	)
	25
	26	// Unit Test for the checkForDuplicates function.
	27	func TestCheckForDuplicates(t *testing.T) {
	28	tables := []struct {
	29	myReq []string
	30	myHeaders map[string]int
	31	myDup map[string]bool
	32	myLow map[string]int
	33	myErr error
	34	}{
	35	{[]string{"name", "id", "last_name", "last_name"}, make(map[string]int), make(map[string]bool), make(map[string]int), ErrAmbiguousFieldName},
	36	{[]string{"name", "id", "last_name", "another_name"}, make(map[string]int), make(map[string]bool), make(map[string]int), nil},
	37	}
	38	for _, table := range tables {
	39	err := checkForDuplicates(table.myReq, table.myHeaders, table.myDup, table.myLow)
	40	if err != table.myErr {
	41	t.Error()
	42	}
	43	}
	44	}
	45
	46	// Test for the function which processes columnnames to make sure that they are
	47	// compatible with spaces.
	48	func TestMyProcessing(t *testing.T) {
	49	options := &Options{
	50	HasHeader: false,
	51	FieldDelimiter: ",",
	52	Comments: "",
	53	Name: "S3Object", // Default table name for all objects
	54	ReadFrom: bytes.NewReader([]byte("Here , is, a, string + \n + random,random,stuff,stuff ")),
	55	Compressed: "",
	56	Expression: "",
	57	OutputFieldDelimiter: ",",
	58	StreamSize: 20,
	59	}
	60	s3s, err := NewInput(options)
	61	if err != nil {
	62	t.Error(err)
	63	}
	64	tables := []struct {
	65	myReq []string
	66	myHeaders map[string]int
	67	myDup map[string]bool
	68	myLow map[string]int
	69	myOpts *Options
	70	input *Input
	71	length int
	72	testOutput string
	73	myErr error
	74	}{
	75	{[]string{"name", "id", "last_name", "CAST"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, s3s, 4, "CAST", nil},
	76	{[]string{"name", "id", "last_name", "another_name"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, s3s, 4, "another_name", nil},
	77	{[]string{"name", "id", "last_name", "another_name"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, s3s, 4, "another_name", nil},
	78	{[]string{"name", "id", "random_name", "fame_name", "another_col"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, s3s, 5, "fame_name", nil},
	79	}
	80	for _, table := range tables {
	81	err = checkForDuplicates(table.myReq, table.myHeaders, table.myDup, table.myLow)
	82	if err != table.myErr {
	83	t.Error()
	84	}
	85	if len(table.myReq) != table.length {
	86	t.Errorf("UnexpectedError")
	87	}
	88	if table.myReq[3] != table.testOutput {
	89	t.Error()
	90	}
	91	}
	92	}
	93
	94	// TestMyRowIndexResults is a unit test which makes sure that the rows that are
	95	// being printed are appropriate to the query being requested.
	96	func TestMyRowIndexResults(t *testing.T) {
	97	options := &Options{
	98	HasHeader: false,
	99	FieldDelimiter: ",",
	100	Comments: "",
	101	Name: "S3Object", // Default table name for all objects
	102	ReadFrom: bytes.NewReader([]byte("Here , is, a, string + \n + random,random,stuff,stuff ")),
	103	Compressed: "",
	104	Expression: "",
	105	OutputFieldDelimiter: ",",
	106	StreamSize: 20,
	107	}
	108	s3s, err := NewInput(options)
	109	if err != nil {
	110	t.Error(err)
	111	}
	112	tables := []struct {
	113	myReq []string
	114	myHeaders map[string]int
	115	myDup map[string]bool
	116	myLow map[string]int
	117	myOpts *Options
	118	input *Input
	119	myRecord []string
	120	myTarget string
	121	myAsterix string
	122	columns []string
	123	err error
	124	}{
	125	{[]string{"1", "2"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, s3s, []string{"target", "random", "hello", "stuff"}, "target,random", "target,random,hello,stuff", []string{"1", "2", "3", "4"}, nil},
	126	{[]string{"2", "3", "4"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, s3s, []string{"random", "hullo", "thing", "stuff"}, "hullo,thing,stuff", "random,hullo,thing,stuff", []string{"1", "2", "3", "4"}, nil},
	127	{[]string{"3", "2"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, s3s, []string{"random", "hullo", "thing", "stuff"}, "thing,hullo", "random,hullo,thing,stuff", []string{"1", "2", "3", "4"}, nil},
	128	{[]string{"11", "1"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, s3s, []string{"random", "hullo", "thing", "stuff"}, "", "random,hullo,thing,stuff", []string{"1", "2", "3", "4"}, ErrInvalidColumnIndex},
	129	}
	130	for _, table := range tables {
	131	checkForDuplicates(table.columns, table.myHeaders, table.myDup, table.myLow)
	132	myRow, err := s3s.processColNameIndex(table.myRecord, table.myReq, table.columns)
	133	if err != table.err {
	134	t.Error()
	135	}
	136	if myRow != table.myTarget {
	137	t.Error()
	138	}
	139	myRow = table.input.printAsterix(table.myRecord)
	140	if myRow != table.myAsterix {
	141	t.Error()
	142	}
	143	}
	144	}
	145
	146	// TestMyHelperFunctions is a unit test which tests some small helper string
	147	// functions.
	148	func TestMyHelperFunctions(t *testing.T) {
	149	tables := []struct {
	150	myReq string
	151	myList []string
	152	myIndex int
	153	expected bool
	154	}{
	155	{"test1", []string{"test1", "test2", "test3", "test4", "test5"}, 0, true},
	156	{"random", []string{"test1", "test2", "test3", "test4", "test5"}, -1, false},
	157	{"test3", []string{"test1", "test2", "test3", "test4", "test5"}, 2, true},
	158	}
	159	for _, table := range tables {
	160	if stringInSlice(table.myReq, table.myList) != table.expected {
	161	t.Error()
	162	}
	163	if stringIndex(table.myReq, table.myList) != table.myIndex {
	164	t.Error()
	165	}
	166	}
	167	}
	168
	169	// TestMyStateMachine is a unit test which ensures that the lowest level of the
	170	// interpreter is converting properly.
	171	func TestMyStateMachine(t *testing.T) {
	172	tables := []struct {
	173	operand interface{}
	174	operator string
	175	leftArg string
	176	err error
	177	expected bool
	178	}{
	179	{"2005", ">", "2012", nil, true},
	180	{2005, ">", "2012", nil, true},
	181	{2012.0000, ">", "2014.000", nil, true},
	182	{"NA", ">", "2014.000", nil, false},
	183	{2014, ">", "Random", nil, false},
	184	{"test3", ">", "aandom", nil, false},
	185	}
	186	for _, table := range tables {
	187	val, err := evaluateOperator(table.leftArg, table.operator, table.operand)
	188	if err != table.err {
	189	t.Error()
	190	}
	191	if val != table.expected {
	192	t.Error()
	193	}
	194	}
	195	}
	196
	197	// TestMyOperators is a unit test which ensures that the appropriate values are
	198	// being returned from the operators functions.
	199	func TestMyOperators(t *testing.T) {
	200	tables := []struct {
	201	operator string
	202	err error
	203	}{
	204	{">", nil},
	205	{"%", ErrParseUnknownOperator},
	206	}
	207	for _, table := range tables {
	208	err := checkValidOperator(table.operator)
	209	if err != table.err {
	210	t.Error()
	211	}
	212	}
	213	}
	214
	215	// TestMyConversion ensures that the conversion of the value from the csv
	216	// happens correctly.
	217	func TestMyConversion(t *testing.T) {
	218	tables := []struct {
	219	myTblVal string
	220	expected reflect.Kind
	221	}{
	222	{"2014", reflect.Int},
	223	{"2014.000", reflect.Float64},
	224	{"String!!!", reflect.String},
	225	}
	226	for _, table := range tables {
	227	val := reflect.ValueOf(checkStringType(table.myTblVal)).Kind()
	228	if val != table.expected {
	229	t.Error()
	230	}
	231	}
	232	}
	233
	234	// Unit Tests for Parser.
	235	func TestMyParser(t *testing.T) {
	236	tables := []struct {
	237	myQuery string
	238	err error
	239	reqCols []string
	240	alias string
	241	myLimit int
	242	aggFuncs []string
	243	header []string
	244	}{
	245	{"SELECT * FROM S3OBJECT", nil, []string{"*"}, "S3OBJECT", 0, make([]string, 1), []string{"name1", "name2", "name3", "name4"}},
	246	{"SELECT * FROM S3OBJECT AS A", nil, []string{"*"}, "A", 0, make([]string, 1), []string{"name1", "name2", "name3", "name4"}},
	247	{"SELECT col_name FROM S3OBJECT AS A", nil, []string{"col_name"}, "A", 0, make([]string, 1), []string{"col_name", "name2", "name3", "name4"}},
	248	{"SELECT col_name,col_other FROM S3OBJECT AS A LIMIT 5", nil, []string{"col_name", "col_other"}, "A", 5, make([]string, 2), []string{"col_name", "col_other", "name3", "name4"}},
	249	{"SELECT col_name,col_other FROM S3OBJECT AS A WHERE col_name = 'Name' LIMIT 5", nil, []string{"col_name", "col_other"}, "A", 5, make([]string, 2), []string{"col_name", "col_other", "name3", "name4"}},
	250	{"SELECT col_name,col_other FROM S3OBJECT AS A WHERE col_name = 'Name LIMIT 5", ErrLexerInvalidChar, nil, "", 0, nil, []string{"col_name", "col_other", "name3", "name4"}},
	251	{"SELECT count() FROM S3OBJECT AS A WHERE col_name = 'Name' LIMIT 5", nil, []string{""}, "A", 5, []string{"count"}, []string{"col_name", "col_other", "name3", "name4"}},
	252	{"SELECT sum(col_name),sum(col_other) FROM S3OBJECT AS A WHERE col_name = 'Name' LIMIT 5", nil, []string{"col_name", "col_other"}, "A", 5, []string{"sum", "sum"}, []string{"col_name", "col_other"}},
	253	{"SELECT A.col_name FROM S3OBJECT AS A", nil, []string{"col_name"}, "A", 0, make([]string, 1), []string{"col_name", "col_other", "name3", "name4"}},
	254	{"SELECT A._col_name FROM S3OBJECT AS A", nil, []string{"col_name"}, "A", 0, make([]string, 1), []string{"col_name", "col_other", "name3", "name4"}},
	255	{"SELECT A._col_name FROM S3OBJECT AS A WHERE randomname > 5", ErrMissingHeaders, nil, "", 0, nil, []string{"col_name", "col_other", "name3", "name4"}},
	256	{"SELECT A._col_name FROM S3OBJECT AS A WHERE A._11 > 5", ErrInvalidColumnIndex, nil, "", 0, nil, []string{"col_name", "col_other", "name3", "name4"}},
	257	{"SELECT COALESCE(col_name,col_other) FROM S3OBJECT AS A WHERE A._3 > 5", nil, []string{""}, "A", 0, []string{""}, []string{"col_name", "col_other", "name3", "name4"}},
	258	{"SELECT COALESCE(col_name,col_other),COALESCE(col_name,col_other) FROM S3OBJECT AS A WHERE A._3 > 5", nil, []string{"", ""}, "A", 0, []string{"", ""}, []string{"col_name", "col_other", "name3", "name4"}},
	259	{"SELECT COALESCE(col_name,col_other) ,col_name , COALESCE(col_name,col_other) FROM S3OBJECT AS A WHERE col_name > 5", nil, []string{"", "col_name", ""}, "A", 0, []string{"", "", ""}, []string{"col_name", "col_other", "name3", "name4"}},
	260	{"SELECT NULLIF(col_name,col_other) ,col_name , COALESCE(col_name,col_other) FROM S3OBJECT AS A WHERE col_name > 5", nil, []string{"", "col_name", ""}, "A", 0, []string{"", "", ""}, []string{"col_name", "col_other", "name3", "name4"}},
	261	{"SELECT NULLIF(col_name,col_other) FROM S3OBJECT AS A WHERE col_name > 5", nil, []string{""}, "A", 0, []string{""}, []string{"col_name", "col_other", "name3", "name4"}},
	262	{"SELECT NULLIF(randomname,col_other) FROM S3OBJECT AS A WHERE col_name > 5", ErrMissingHeaders, nil, "", 0, nil, []string{"col_name", "col_other", "name3", "name4"}},
	263	{"SELECT col_name FROM S3OBJECT AS A WHERE COALESCE(random,5) > 5", ErrMissingHeaders, nil, "", 0, nil, []string{"col_name", "col_other", "name3", "name4"}},
	264	{"SELECT col_name FROM S3OBJECT AS A WHERE NULLIF(random,5) > 5", ErrMissingHeaders, nil, "", 0, nil, []string{"col_name", "col_other", "name3", "name4"}},
	265	{"SELECT col_name FROM S3OBJECT AS A WHERE LOWER(col_name) BETWEEN 5 AND 7", nil, []string{"col_name"}, "A", 0, []string{""}, []string{"col_name", "col_other", "name3", "name4"}},
	266	{"SELECT UPPER(col_name) FROM S3OBJECT AS A WHERE LOWER(col_name) BETWEEN 5 AND 7", nil, []string{""}, "A", 0, []string{""}, []string{"col_name", "col_other", "name3", "name4"}},
	267	{"SELECT UPPER(*) FROM S3OBJECT AS A WHERE LOWER(col_name) BETWEEN 5 AND 7", ErrParseUnsupportedCallWithStar, nil, "", 0, nil, []string{"col_name", "col_other", "name3", "name4"}},
	268	{"SELECT NULLIF(col_name,col_name) FROM S3OBJECT AS A WHERE NULLIF(LOWER(col_name),col_name) BETWEEN 5 AND 7", nil, []string{""}, "A", 0, []string{""}, []string{"col_name", "col_other", "name3", "name4"}},
	269	{"SELECT COALESCE(col_name,col_name) FROM S3OBJECT AS A WHERE NULLIF(LOWER(col_name),col_name) BETWEEN 5 AND 7", nil, []string{""}, "A", 0, []string{""}, []string{"col_name", "col_other", "name3", "name4"}},
	270	}
	271	for _, table := range tables {
	272	options := &Options{
	273	HasHeader: false,
	274	FieldDelimiter: ",",
	275	Comments: "",
	276	Name: "S3Object", // Default table name for all objects
	277	ReadFrom: bytes.NewReader([]byte("name1,name2,name3,name4" + "\n" + "5,is,a,string" + "\n" + "random,random,stuff,stuff")),
	278	Compressed: "",
	279	Expression: "",
	280	OutputFieldDelimiter: ",",
	281	StreamSize: 20,
	282	HeaderOpt: true,
	283	}
	284	s3s, err := NewInput(options)
	285	if err != nil {
	286	t.Error(err)
	287	}
	288	s3s.header = table.header
	289	reqCols, alias, myLimit, _, aggFunctionNames, _, err := s3s.ParseSelect(table.myQuery)
	290	if table.err != err {
	291	t.Error()
	292	}
	293	if !reflect.DeepEqual(reqCols, table.reqCols) {
	294	t.Error()
	295	}
	296	if alias != table.alias {
	297	t.Error()
	298	}
	299	if myLimit != table.myLimit {
	300	t.Error()
	301	}
	302	if !reflect.DeepEqual(table.aggFuncs, aggFunctionNames) {
	303	t.Error()
	304	}
	305	}
	306	}
	307
	308	// Unit tests for the main function that performs aggreggation.
	309	func TestMyAggregationFunc(t *testing.T) {
	310	columnsMap := make(map[string]int)
	311	columnsMap["Col1"] = 0
	312	columnsMap["Col2"] = 1
	313	tables := []struct {
	314	counter int
	315	filtrCount int
	316	myAggVals []float64
	317	columnsMap map[string]int
	318	storeReqCols []string
	319	storeFunctions []string
	320	record []string
	321	err error
	322	expectedVal float64
	323	}{
	324	{10, 5, []float64{10}, columnsMap, []string{"Col1"}, []string{"count"}, []string{"1", "2"}, nil, 11},
	325	{10, 5, []float64{10}, columnsMap, []string{"Col1"}, []string{"min"}, []string{"1", "2"}, nil, 1},
	326	{10, 5, []float64{10}, columnsMap, []string{"Col1"}, []string{"max"}, []string{"1", "2"}, nil, 10},
	327	{10, 5, []float64{10}, columnsMap, []string{"Col1"}, []string{"sum"}, []string{"1", "2"}, nil, 11},
	328	{1, 1, []float64{10}, columnsMap, []string{"Col1"}, []string{"avg"}, []string{"1", "2"}, nil, 5.500},
	329	{10, 5, []float64{0.000}, columnsMap, []string{"Col1"}, []string{"random"}, []string{"1", "2"}, ErrParseNonUnaryAgregateFunctionCall, 0},
	330	{0, 5, []float64{0}, columnsMap, []string{"0"}, []string{"count"}, []string{"1", "2"}, nil, 1},
	331	{10, 5, []float64{10}, columnsMap, []string{"1"}, []string{"min"}, []string{"1", "12"}, nil, 10},
	332	}
	333	for _, table := range tables {
	334	err := aggregationFunctions(table.counter, table.filtrCount, table.myAggVals, table.columnsMap, table.storeReqCols, table.storeFunctions, table.record)
	335	if table.err != err {
	336	t.Error()
	337	}
	338	if table.myAggVals[0] != table.expectedVal {
	339	t.Error()
	340	}
	341
	342	}
	343	}
	344
	345	// Unit Tests for the function which converts a float array to string.
	346	func TestToStringAgg(t *testing.T) {
	347	options := &Options{
	348	HasHeader: false,
	349	FieldDelimiter: ",",
	350	Comments: "",
	351	Name: "S3Object", // Default table name for all objects
	352	ReadFrom: bytes.NewReader([]byte("Here , is, a, string + \n + random,random,stuff,stuff ")),
	353	Compressed: "",
	354	Expression: "",
	355	OutputFieldDelimiter: ",",
	356	StreamSize: 20,
	357	HeaderOpt: true,
	358	}
	359	s3s, err := NewInput(options)
	360	if err != nil {
	361	t.Error(err)
	362	}
	363	tables := []struct {
	364	myAggVal []float64
	365	expected string
	366	}{
	367	{[]float64{10, 11, 12, 13, 14}, "10.000000,11.000000,12.000000,13.000000,14.000000"},
	368	{[]float64{10}, "10.000000"},
	369	}
	370	for _, table := range tables {
	371	val := s3s.aggFuncToStr(table.myAggVal)
	372	if val != table.expected {
	373	t.Error()
	374	}
	375	}
	376	}
	377
	378	// TestMyRowColLiteralResults is a unit test which makes sure that the rows that
	379	// are being printed are appropriate to the query being requested.
	380	func TestMyRowColLiteralResults(t *testing.T) {
	381	options := &Options{
	382	HasHeader: false,
	383	FieldDelimiter: ",",
	384	Comments: "",
	385	Name: "S3Object", // Default table name for all objects
	386	ReadFrom: bytes.NewReader([]byte("Here , is, a, string + \n + random,random,stuff,stuff ")),
	387	Compressed: "",
	388	Expression: "",
	389	OutputFieldDelimiter: ",",
	390	StreamSize: 20,
	391	HeaderOpt: true,
	392	}
	393	s3s, err := NewInput(options)
	394	if err != nil {
	395	t.Error(err)
	396	}
	397	tables := []struct {
	398	myReq []string
	399	myHeaders map[string]int
	400	myDup map[string]bool
	401	myLow map[string]int
	402	myOpts *Options
	403	tempList []string
	404	input *Input
	405	myRecord []string
	406	myTarget string
	407	columns []string
	408	err error
	409	}{
	410	{[]string{"draft", "year"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, []string{"draft", "year"}, s3s, []string{"target", "random", "hello", "stuff"}, "target,random", []string{"draft", "year", "random", "another"}, nil},
	411	{[]string{"year", "draft"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, []string{"year", "draft"}, s3s, []string{"draft", "2012", "thing", "stuff"}, "2012,draft", []string{"draft", "year", "random", "another"}, nil},
	412	{[]string{"yearrandomstuff", "draft"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, []string{"yearrandomstuff", "draft"}, s3s, []string{"draft", "2012", "thing", "stuff"}, "", []string{"draft", "year", "random", "another"}, ErrMissingHeaders},
	413	{[]string{"draft", "randomstuff"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, []string{"yearrandomstuff", "draft"}, s3s, []string{"draft", "2012", "thing", "stuff"}, "", []string{"draft", "year", "random", "another"}, ErrMissingHeaders},
	414	}
	415	for _, table := range tables {
	416	checkForDuplicates(table.columns, table.myHeaders, table.myDup, table.myLow)
	417	myRow, err := table.input.processColNameLiteral(table.myRecord, table.myReq, table.tempList, table.myHeaders, nil)
	418	if err != table.err {
	419	t.Error()
	420	}
	421	if myRow != table.myTarget {
	422	t.Error()
	423	}
	424	}
	425	}
	426
	427	// TestMyWhereEval is a function which provides unit tests for the function
	428	// which evaluates the where clause.
	429	func TestMyWhereEval(t *testing.T) {
	430	columnsMap := make(map[string]int)
	431	columnsMap["Col1"] = 0
	432	columnsMap["Col2"] = 1
	433	tables := []struct {
	434	myQuery string
	435	record []string
	436	err error
	437	expected bool
	438	header []string
	439	}{
	440	{"SELECT * FROM S3OBJECT", []string{"record_1,record_2,record_3,record_4"}, nil, true, []string{"Col1", "Col2"}},
	441	{"SELECT * FROM S3OBJECT WHERE Col1 < -1", []string{"0", "1"}, nil, false, []string{"Col1", "Col2"}},
	442	{"SELECT * FROM S3OBJECT WHERE Col1 < -1 OR Col2 > 15", []string{"151", "12"}, nil, false, []string{"Col1", "Col2"}},
	443	{"SELECT * FROM S3OBJECT WHERE Col1 > -1 AND Col2 > 15", []string{"151", "12"}, nil, false, []string{"Col1", "Col2"}},
	444	{"SELECT * FROM S3OBJECT WHERE Col1 > 1.00", []string{"151.0000", "12"}, nil, true, []string{"Col1", "Col2"}},
	445	{"SELECT * FROM S3OBJECT WHERE Col1 > 100", []string{"random", "12"}, nil, false, []string{"Col1", "Col2"}},
	446	{"SELECT * FROM S3OBJECT WHERE Col1 BETWEEN 100 AND 0", []string{"151", "12"}, nil, false, []string{"Col1", "Col2"}},
	447	{"SELECT * FROM S3OBJECT WHERE Col1 BETWEEN 100.0 AND 0.0", []string{"151", "12"}, nil, false, []string{"Col1", "Col2"}},
	448	{"SELECT * FROM S3OBJECT AS A WHERE A.1 BETWEEN 160 AND 150", []string{"151", "12"}, nil, true, []string{"Col1", "Col2"}},
	449	{"SELECT * FROM S3OBJECT AS A WHERE A._1 BETWEEN 160 AND 0", []string{"151", "12"}, nil, true, []string{"Col1", "Col2"}},
	450	{"SELECT * FROM S3OBJECT AS A WHERE A._1 BETWEEN 0 AND 160", []string{"151", "12"}, nil, true, []string{"Col1", "Col2"}},
	451	{"SELECT * FROM S3OBJECT A._1 LIKE 'r%'", []string{"record_1,record_2,record_3,record_4"}, nil, true, []string{"Col1", "Col2"}},
	452	{"SELECT s._2 FROM S3Object s WHERE s._2 = 'Steven'", []string{"record_1", "Steven", "Steven", "record_4"}, nil, true, []string{"Col1", "Col2"}},
	453	{"SELECT * FROM S3OBJECT AS A WHERE Col1 BETWEEN 0 AND 160", []string{"151", "12"}, nil, true, []string{"Col1", "Col2"}},
	454	{"SELECT * FROM S3OBJECT AS A WHERE Col1 BETWEEN 160 AND 0", []string{"151", "12"}, nil, true, []string{"Col1", "Col2"}},
	455	{"SELECT * FROM S3OBJECT AS A WHERE UPPER(Col1) BETWEEN 160 AND 0", []string{"151", "12"}, nil, true, []string{"Col1", "Col2"}},
	456	{"SELECT * FROM S3OBJECT AS A WHERE UPPER(Col1) = 'RANDOM'", []string{"random", "12"}, nil, true, []string{"Col1", "Col2"}},
	457	{"SELECT * FROM S3OBJECT AS A WHERE LOWER(UPPER(Col1) = 'random'", []string{"random", "12"}, nil, true, []string{"Col1", "Col2"}},
	458	}
	459	for _, table := range tables {
	460	options := &Options{
	461	HasHeader: false,
	462	FieldDelimiter: ",",
	463	Comments: "",
	464	Name: "S3Object", // Default table name for all objects
	465	ReadFrom: bytes.NewReader([]byte("name1,name2,name3,name4" + "\n" + "5,is,a,string" + "\n" + "random,random,stuff,stuff")),
	466	Compressed: "",
	467	Expression: "",
	468	OutputFieldDelimiter: ",",
	469	StreamSize: 20,
	470	HeaderOpt: true,
	471	}
	472	s3s, err := NewInput(options)
	473	s3s.header = table.header
	474
	475	if err != nil {
	476	t.Error(err)
	477	}
	478	_, alias, _, whereClause, _, _, _ := s3s.ParseSelect(table.myQuery)
	479	myVal, err := matchesMyWhereClause(table.record, columnsMap, alias, whereClause)
	480	if table.err != err {
	481	t.Error()
	482	}
	483	if myVal != table.expected {
	484	t.Error()
	485	}
	486	}
	487	}
	488
	489	// TestMyStringComparator is a unit test which ensures that the appropriate
	490	// values are being compared for strings.
	491	func TestMyStringComparator(t *testing.T) {
	492	tables := []struct {
	493	operand string
	494	operator string
	495	myVal string
	496	expected bool
	497	err error
	498	}{
	499	{"random", ">", "myName", "random" > "myName", nil},
	500	{"12", "!=", "myName", "12" != "myName", nil},
	501	{"12", "=", "myName", "12" == "myName", nil},
	502	{"12", "<=", "myName", "12" <= "myName", nil},
	503	{"12", ">=", "myName", "12" >= "myName", nil},
	504	{"12", "<", "myName", "12" < "myName", nil},
	505	{"name", "like", "_x%", false, nil},
	506	{"12", "randomoperator", "myName", false, ErrUnsupportedSyntax},
	507	}
	508	for _, table := range tables {
	509	myVal, err := stringEval(table.operand, table.operator, table.myVal)
	510	if err != table.err {
	511	t.Error()
	512	}
	513	if myVal != table.expected {
	514	t.Error()
	515	}
	516	}
	517	}
	518
	519	// TestMyFloatComparator is a unit test which ensures that the appropriate
	520	// values are being compared for floats.
	521	func TestMyFloatComparator(t *testing.T) {
	522	tables := []struct {
	523	operand float64
	524	operator string
	525	myVal float64
	526	expected bool
	527	err error
	528	}{
	529	{12.000, ">", 13.000, 12.000 > 13.000, nil},
	530	{1000.000, "!=", 1000.000, 1000.000 != 1000.000, nil},
	531	{1000.000, "<", 1000.000, 1000.000 < 1000.000, nil},
	532	{1000.000, "<=", 1000.000, 1000.000 <= 1000.000, nil},
	533	{1000.000, ">=", 1000.000, 1000.000 >= 1000.000, nil},
	534	{1000.000, "=", 1000.000, 1000.000 == 1000.000, nil},
	535	{17.000, "randomoperator", 0.0, false, ErrUnsupportedSyntax},
	536	}
	537	for _, table := range tables {
	538	myVal, err := floatEval(table.operand, table.operator, table.myVal)
	539	if err != table.err {
	540	t.Error()
	541	}
	542	if myVal != table.expected {
	543	t.Error()
	544	}
	545	}
	546	}
	547
	548	// TestMyIntComparator is a unit test which ensures that the appropriate values
	549	// are being compared for ints.
	550	func TestMyIntComparator(t *testing.T) {
	551	tables := []struct {
	552	operand int64
	553	operator string
	554	myVal int64
	555	expected bool
	556	err error
	557	}{
	558	{12, ">", 13, 12.000 > 13.000, nil},
	559	{1000, "!=", 1000, 1000.000 != 1000.000, nil},
	560	{1000, "<", 1000, 1000.000 < 1000.000, nil},
	561	{1000, "<=", 1000, 1000.000 <= 1000.000, nil},
	562	{1000, ">=", 1000, 1000.000 >= 1000.000, nil},
	563	{1000, "=", 1000, 1000.000 >= 1000.000, nil},
	564	{17, "randomoperator", 0, false, ErrUnsupportedSyntax},
	565	}
	566	for _, table := range tables {
	567	myVal, err := intEval(table.operand, table.operator, table.myVal)
	568	if err != table.err {
	569	t.Error()
	570	}
	571	if myVal != table.expected {
	572	t.Error()
	573	}
	574	}
	575	}
	576
	577	// TestMySizeFunction is a function which provides unit testing for the function
	578	// which calculates size.
	579	func TestMySizeFunction(t *testing.T) {
	580	tables := []struct {
	581	myRecord []string
	582	expected int64
	583	}{
	584	{[]string{"test1", "test2", "test3", "test4", "test5"}, 30},
	585	}
	586	for _, table := range tables {
	587	if processSize(table.myRecord) != table.expected {
	588	t.Error()
	589	}
	590
	591	}
	592	}
	593
	594	// TestInterpreter is a function which provides unit testing for the main
	595	// interpreter function.
	596	func TestInterpreter(t *testing.T) {
	597	tables := []struct {
	598	myQuery string
	599	myChan chan *Row
	600	err error
	601	header []string
	602	}{
	603	{"Select random from S3OBJECT", make(chan *Row), ErrMissingHeaders, []string{"name1", "name2", "name3", "name4"}},
	604	{"Select * from S3OBJECT as A WHERE name2 > 5.00", make(chan *Row), nil, []string{"name1", "name2", "name3", "name4"}},
	605	{"Select * from S3OBJECT", make(chan *Row), nil, []string{"name1", "name2", "name3", "name4"}},
	606	{"Select A_1 from S3OBJECT as A", make(chan *Row), nil, []string{"1", "2", "3", "4"}},
	607	{"Select count() from S3OBJECT", make(chan Row), nil, []string{"name1", "name2", "name3", "name4"}},
	608	{"Select * from S3OBJECT WHERE name1 > 5.00", make(chan *Row), nil, []string{"name1", "name2", "name3", "name4"}},
	609	}
	610	for _, table := range tables {
	611	options := &Options{
	612	HasHeader: false,
	613	FieldDelimiter: ",",
	614	Comments: "",
	615	Name: "S3Object", // Default table name for all objects
	616	ReadFrom: bytes.NewReader([]byte("name1,name2,name3,name4" + "\n" + "5,is,a,string" + "\n" + "random,random,stuff,stuff")),
	617	Compressed: "",
	618	Expression: "",
	619	OutputFieldDelimiter: ",",
	620	StreamSize: 20,
	621	HeaderOpt: true,
	622	}
	623	s3s, err := NewInput(options)
	624	if err != nil {
	625	t.Error(err)
	626	}
	627	s3s.header = table.header
	628	reqCols, alias, myLimit, whereClause, aggFunctionNames, _, err := s3s.ParseSelect(table.myQuery)
	629	if err != table.err {
	630	t.Fatal()
	631	}
	632	if err == nil {
	633	go s3s.processSelectReq(reqCols, alias, whereClause, myLimit, aggFunctionNames, table.myChan, nil)
	634	select {
	635	case row, ok := <-table.myChan:
	636	if ok && len(row.record) > 0 {
	637	} else if ok && row.err != nil {
	638	if row.err != table.err {
	639	t.Error()
	640	}
	641	close(table.myChan)
	642	} else if !ok {
	643	}
	644	}
	645	}
	646	}
	647	}
	648
	649	// TestMyXMLFunction is a function that provides unit testing for the XML
	650	// creating function.
	651	func TestMyXMLFunction(t *testing.T) {
	652	options := &Options{
	653	HasHeader: false,
	654	FieldDelimiter: ",",
	655	Comments: "",
	656	Name: "S3Object", // Default table name for all objects
	657	ReadFrom: bytes.NewReader([]byte("name1,name2,name3,name4" + "\n" + "5,is,a,string" + "\n" + "random,random,stuff,stuff")),
	658	Compressed: "",
	659	Expression: "",
	660	OutputFieldDelimiter: ",",
	661	StreamSize: 20,
	662	HeaderOpt: true,
	663	}
	664	s3s, err := NewInput(options)
	665	if err != nil {
	666	t.Error(err)
	667	}
	668	tables := []struct {
	669	expectedStat int
	670	expectedProgress int
	671	}{
	672	{159, 165},
	673	}
	674	for _, table := range tables {
	675	myVal, _ := s3s.createStatXML()
	676	myOtherVal, _ := s3s.createProgressXML()
	677	if len(myVal) != table.expectedStat {
	678	t.Error()
	679	}
	680	if len(myOtherVal) != table.expectedProgress {
	681	fmt.Println(len(myOtherVal))
	682	t.Error()
	683	}
	684	}
	685	}
	686
	687	// TestMyProtocolFunction is a function which provides unit testing for several
	688	// of the functions which write the binary protocol.
	689	func TestMyProtocolFunction(t *testing.T) {
	690	options := &Options{
	691	HasHeader: false,
	692	FieldDelimiter: ",",
	693	Comments: "",
	694	Name: "S3Object", // Default table name for all objects
	695	ReadFrom: bytes.NewReader([]byte("name1,name2,name3,name4" + "\n" + "5,is,a,string" + "\n" + "random,random,stuff,stuff")),
	696	Compressed: "",
	697	Expression: "",
	698	OutputFieldDelimiter: ",",
	699	StreamSize: 20,
	700	HeaderOpt: true,
	701	}
	702	s3s, err := NewInput(options)
	703	if err != nil {
	704	t.Error(err)
	705	}
	706	tables := []struct {
	707	payloadMsg string
	708	expectedRecord int
	709	expectedEnd int
	710	}{
	711	{"random payload", 115, 56},
	712	}
	713	for _, table := range tables {
	714	var currentMessage = &bytes.Buffer{}
	715	if len(s3s.writeRecordMessage(table.payloadMsg, currentMessage).Bytes()) != table.expectedRecord {
	716	t.Error()
	717	}
	718	currentMessage.Reset()
	719	if len(s3s.writeEndMessage(currentMessage).Bytes()) != table.expectedEnd {
	720	t.Error()
	721	}
	722	currentMessage.Reset()
	723	if len(s3s.writeContinuationMessage(currentMessage).Bytes()) != 57 {
	724	t.Error()
	725	}
	726	currentMessage.Reset()
	727	}
	728	}
	729
	730	// TestMyInfoProtocolFunctions is a function which provides unit testing for the
	731	// stat and progress messages of the protocols.
	732	func TestMyInfoProtocolFunctions(t *testing.T) {
	733	options := &Options{
	734	HasHeader: true,
	735	FieldDelimiter: ",",
	736	Comments: "",
	737	Name: "S3Object", // Default table name for all objects
	738	ReadFrom: bytes.NewReader([]byte("name1,name2,name3,name4" + "\n" + "5,is,a,string" + "\n" + "random,random,stuff,stuff")),
	739	Compressed: "",
	740	Expression: "",
	741	OutputFieldDelimiter: ",",
	742	StreamSize: 20,
	743	}
	744	s3s, err := NewInput(options)
	745	if err != nil {
	746	t.Error(err)
	747	}
	748	myVal, _ := s3s.createStatXML()
	749	myOtherVal, _ := s3s.createProgressXML()
	750
	751	tables := []struct {
	752	payloadStatMsg string
	753	payloadProgressMsg string
	754	expectedStat int
	755	expectedProgress int
	756	}{
	757	{myVal, myOtherVal, 242, 252},
	758	}
	759	for _, table := range tables {
	760	var currBuf = &bytes.Buffer{}
	761	if len(s3s.writeStatMessage(table.payloadStatMsg, currBuf).Bytes()) != table.expectedStat {
	762	t.Error()
	763	}
	764	currBuf.Reset()
	765	if len(s3s.writeProgressMessage(table.payloadProgressMsg, currBuf).Bytes()) != table.expectedProgress {
	766	t.Error()
	767	}
	768	}
	769	}
	770
	771	// TestMyErrorProtocolFunctions is a function which provides unit testing for
	772	// the error message type of protocol.
	773	func TestMyErrorProtocolFunctions(t *testing.T) {
	774	options := &Options{
	775	HasHeader: false,
	776	FieldDelimiter: ",",
	777	Comments: "",
	778	Name: "S3Object", // Default table name for all objects
	779	ReadFrom: bytes.NewReader([]byte("name1,name2,name3,name4" + "\n" + "5,is,a,string" + "\n" + "random,random,stuff,stuff")),
	780	Compressed: "",
	781	Expression: "",
	782	OutputFieldDelimiter: ",",
	783	StreamSize: 20,
	784	HeaderOpt: true,
	785	}
	786	s3s, err := NewInput(options)
	787	if err != nil {
	788	t.Error(err)
	789	}
	790	tables := []struct {
	791	err error
	792	expectedError int
	793	}{
	794	{ErrInvalidCast, 248},
	795	{ErrTruncatedInput, 200},
	796	{ErrUnsupportedSyntax, 114},
	797	{ErrCSVParsingError, 157},
	798	}
	799	for _, table := range tables {
	800	var currentMessage = &bytes.Buffer{}
	801	if len(s3s.writeErrorMessage(table.err, currentMessage).Bytes()) != table.expectedError {
	802	t.Error()
	803	}
	804
	805	}
	806	}
	807	func TestMatch(t *testing.T) {
	808	testCases := []struct {
	809	pattern string
	810	text string
	811	matched bool
	812	}{
	813	// Test case - 1.
	814	// Test case so that the match occurs on the opening letter.
	815	{
	816	pattern: "a%",
	817	text: "apple",
	818	matched: true,
	819	},
	820	// Test case - 2.
	821	// Test case so that the ending letter is true.
	822	{
	823	pattern: "%m",
	824	text: "random",
	825	matched: true,
	826	},
	827	// Test case - 3.
	828	// Test case so that a character is at the appropriate position.
	829	{
	830	pattern: "_d%",
	831	text: "adam",
	832	matched: true,
	833	},
	834	// Test case - 4.
	835	// Test case so that a character is at the appropriate position.
	836	{
	837	pattern: "_d%",
	838	text: "apple",
	839	matched: false,
	840	},
	841	// Test case - 5.
	842	// Test case with checking that it is at least 3 in length
	843	{
	844	pattern: "a_%_%",
	845	text: "ap",
	846	matched: false,
	847	},
	848	{
	849	pattern: "a_%_%",
	850	text: "apple",
	851	matched: true,
	852	},
	853	{
	854	pattern: "%or%",
	855	text: "orphan",
	856	matched: true,
	857	},
	858	{
	859	pattern: "%or%",
	860	text: "dolphin",
	861	matched: false,
	862	},
	863	{
	864	pattern: "%or%",
	865	text: "dorlphin",
	866	matched: true,
	867	},
	868	{
	869	pattern: "2__3",
	870	text: "2003",
	871	matched: true,
	872	},
	873	{
	874	pattern: "_YYYY_",
	875	text: "aYYYYa",
	876	matched: true,
	877	},
	878	{
	879	pattern: "C%",
	880	text: "CA",
	881	matched: true,
	882	},
	883	{
	884	pattern: "C%",
	885	text: "SC",
	886	matched: false,
	887	},
	888	{
	889	pattern: "%C",
	890	text: "SC",
	891	matched: true,
	892	},
	893	{
	894	pattern: "%C",
	895	text: "CA",
	896	matched: false,
	897	},
	898	{
	899	pattern: "%C",
	900	text: "ACCC",
	901	matched: true,
	902	},
	903	{
	904	pattern: "C%",
	905	text: "CCC",
	906	matched: true,
	907	},
	908	{
	909	pattern: "j%",
	910	text: "mejri",
	911	matched: false,
	912	},
	913	{
	914	pattern: "a%o",
	915	text: "ando",
	916	matched: true,
	917	},
	918	{
	919	pattern: "%j",
	920	text: "mejri",
	921	matched: false,
	922	},
	923	{
	924	pattern: "%ja",
	925	text: "mejrija",
	926	matched: true,
	927	},
	928	{
	929	pattern: "ja%",
	930	text: "jamal",
	931	matched: true,
	932	},
	933	{
	934	pattern: "a%o",
	935	text: "andp",
	936	matched: false,
	937	},
	938	{
	939	pattern: "_r%",
	940	text: "arpa",
	941	matched: true,
	942	},
	943	{
	944	pattern: "_r%",
	945	text: "apra",
	946	matched: false,
	947	},
	948	{
	949	pattern: "a_%_%",
	950	text: "appple",
	951	matched: true,
	952	},
	953	{
	954	pattern: "l_b%",
	955	text: "lebron",
	956	matched: true,
	957	},
	958	{
	959	pattern: "leb%",
	960	text: "Dalembert",
	961	matched: false,
	962	},
	963	{
	964	pattern: "leb%",
	965	text: "Landesberg",
	966	matched: false,
	967	},
	968	{
	969	pattern: "leb%",
	970	text: "Mccalebb",
	971	matched: false,
	972	},
	973	{
	974	pattern: "%lebb",
	975	text: "Mccalebb",
	976	matched: true,
	977	},
	978	}
	979	// Iterating over the test cases, call the function under test and asert the output.
	980	for i, testCase := range testCases {
	981	actualResult, err := likeConvert(testCase.pattern, testCase.text)
	982	if err != nil {
	983	t.Error()
	984	}
	985	if testCase.matched != actualResult {
	986	fmt.Println("Expected Pattern", testCase.pattern, "Expected Text", testCase.text)
	987	t.Errorf("Test %d: Expected the result to be `%v`, but instead found it to be `%v`", i+1, testCase.matched, actualResult)
	988	}
	989	}
	990	}
	991
	992	// TestMyValids is a unit test which ensures that the appropriate values are
	993	// being returned from the isValid... functions.
	994	func TestMyValids(t *testing.T) {
	995
	996	tables := []struct {
	997	myQuery string
	998	indexList []int
	999	myIndex int
	1000	myValIndex bool
	1001	header []string
	1002	err error
	1003	}{
	1004	{"SELECT UPPER(NULLIF(draft_year,random_name))", []int{3, 5, 6, 7, 8, 9}, 3, true, []string{"draft_year", "random_name"}, nil},
	1005	{"SELECT UPPER(NULLIF(draft_year,xandom_name))", []int{3, 5, 6, 7, 8, 9}, 3, true, []string{"draft_year", "random_name"}, ErrMissingHeaders},
	1006	}
	1007	for _, table := range tables {
	1008	options := &Options{
	1009	HasHeader: false,
	1010	FieldDelimiter: ",",
	1011	Comments: "",
	1012	Name: "S3Object", // Default table name for all objects
	1013	ReadFrom: bytes.NewReader([]byte("name1,name2,name3,name4" + "\n" + "5,is,a,string" + "\n" + "random,random,stuff,stuff")),
	1014	Compressed: "",
	1015	Expression: "",
	1016	OutputFieldDelimiter: ",",
	1017	StreamSize: 20,
	1018	HeaderOpt: true,
	1019	}
	1020	s3s, err := NewInput(options)
	1021	if err != nil {
	1022	t.Error(err)
	1023	}
	1024	s3s.header = table.header
	1025	_, _, _, _, _, _, err = s3s.ParseSelect(table.myQuery)
	1026	if err != table.err {
	1027	t.Fatal()
	1028	}
	1029	myVal := isValidFunc(table.indexList, table.myIndex)
	1030	if myVal != table.myValIndex {
	1031	t.Error()
	1032	}
	1033	}
	1034	}
	1035
	1036	// TestMyFuncProcessing is a unit test which ensures that the appropriate values are
	1037	// being returned from the Processing... functions.
	1038	func TestMyFuncProcessing(t *testing.T) {
	1039	tables := []struct {
	1040	myString string
	1041	nullList []string
	1042	coalList []string
	1043	myValString string
	1044	myValCoal string
	1045	myValNull string
	1046	stringFunc string
	1047	}{
	1048	{"lower", []string{"yo", "yo"}, []string{"random", "hello", "random"}, "LOWER", "random", "", "UPPER"},
	1049	{"LOWER", []string{"null", "random"}, []string{"missing", "hello", "random"}, "lower", "hello", "null", "LOWER"},
	1050	}
	1051	for _, table := range tables {
	1052	if table.coalList != nil {
	1053	myVal := processCoalNoIndex(table.coalList)
	1054	if myVal != table.myValCoal {
	1055	t.Error()
	1056	}
	1057	}
	1058	if table.nullList != nil {
	1059	myVal := processNullIf(table.nullList)
	1060	if myVal != table.myValNull {
	1061	t.Error()
	1062	}
	1063	}
	1064	myVal := applyStrFunc(table.myString, table.stringFunc)
	1065	if myVal != table.myValString {
	1066	t.Error()
	1067	}
	1068
	1069	}
	1070	}

vendor/github.com/klauspost/compress/LICENSE+27−0

@@ -0,0 +1,27 @@
	1	Copyright (c) 2012 The Go Authors. All rights reserved.
	2
	3	Redistribution and use in source and binary forms, with or without
	4	modification, are permitted provided that the following conditions are
	5	met:
	6
	7	* Redistributions of source code must retain the above copyright
	8	notice, this list of conditions and the following disclaimer.
	9	* Redistributions in binary form must reproduce the above
	10	copyright notice, this list of conditions and the following disclaimer
	11	in the documentation and/or other materials provided with the
	12	distribution.
	13	* Neither the name of Google Inc. nor the names of its
	14	contributors may be used to endorse or promote products derived from
	15	this software without specific prior written permission.
	16
	17	THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	18	"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	19	LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	20	A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	21	OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	22	SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	23	LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	24	DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	25	THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	26	(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	27	OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/klauspost/compress/flate/copy.go+32−0

@@ -0,0 +1,32 @@
	1	// Copyright 2012 The Go Authors. All rights reserved.
	2	// Use of this source code is governed by a BSD-style
	3	// license that can be found in the LICENSE file.
	4
	5	package flate
	6
	7	// forwardCopy is like the built-in copy function except that it always goes
	8	// forward from the start, even if the dst and src overlap.
	9	// It is equivalent to:
	10	// for i := 0; i < n; i++ {
	11	// mem[dst+i] = mem[src+i]
	12	// }
	13	func forwardCopy(mem []byte, dst, src, n int) {
	14	if dst <= src {
	15	copy(mem[dst:dst+n], mem[src:src+n])
	16	return
	17	}
	18	for {
	19	if dst >= src+n {
	20	copy(mem[dst:dst+n], mem[src:src+n])
	21	return
	22	}
	23	// There is some forward overlap. The destination
	24	// will be filled with a repeated pattern of mem[src:src+k].
	25	// We copy one instance of the pattern here, then repeat.
	26	// Each time around this loop k will double.
	27	k := dst - src
	28	copy(mem[dst:dst+k], mem[src:src+k])
	29	n -= k
	30	dst += k
	31	}
	32	}

vendor/github.com/klauspost/compress/flate/crc32_amd64.go+41−0

@@ -0,0 +1,41 @@
	1	//+build !noasm
	2	//+build !appengine
	3
	4	// Copyright 2015, Klaus Post, see LICENSE for details.
	5
	6	package flate
	7
	8	import (
	9	"github.com/klauspost/cpuid"
	10	)
	11
	12	// crc32sse returns a hash for the first 4 bytes of the slice
	13	// len(a) must be >= 4.
	14	//go:noescape
	15	func crc32sse(a []byte) uint32
	16
	17	// crc32sseAll calculates hashes for each 4-byte set in a.
	18	// dst must be east len(a) - 4 in size.
	19	// The size is not checked by the assembly.
	20	//go:noescape
	21	func crc32sseAll(a []byte, dst []uint32)
	22
	23	// matchLenSSE4 returns the number of matching bytes in a and b
	24	// up to length 'max'. Both slices must be at least 'max'
	25	// bytes in size.
	26	//
	27	// TODO: drop the "SSE4" name, since it doesn't use any SSE instructions.
	28	//
	29	//go:noescape
	30	func matchLenSSE4(a, b []byte, max int) int
	31
	32	// histogram accumulates a histogram of b in h.
	33	// h must be at least 256 entries in length,
	34	// and must be cleared before calling this function.
	35	//go:noescape
	36	func histogram(b []byte, h []int32)
	37
	38	// Detect SSE 4.2 feature.
	39	func init() {
	40	useSSE42 = cpuid.CPU.SSE42()
	41	}

vendor/github.com/klauspost/compress/flate/crc32_amd64.s+213−0

@@ -0,0 +1,213 @@
	1	//+build !noasm
	2	//+build !appengine
	3
	4	// Copyright 2015, Klaus Post, see LICENSE for details.
	5
	6	// func crc32sse(a []byte) uint32
	7	TEXT ·crc32sse(SB), 4, $0
	8	MOVQ a+0(FP), R10
	9	XORQ BX, BX
	10
	11	// CRC32 dword (R10), EBX
	12	BYTE $0xF2; BYTE $0x41; BYTE $0x0f
	13	BYTE $0x38; BYTE $0xf1; BYTE $0x1a
	14
	15	MOVL BX, ret+24(FP)
	16	RET
	17
	18	// func crc32sseAll(a []byte, dst []uint32)
	19	TEXT ·crc32sseAll(SB), 4, $0
	20	MOVQ a+0(FP), R8 // R8: src
	21	MOVQ a_len+8(FP), R10 // input length
	22	MOVQ dst+24(FP), R9 // R9: dst
	23	SUBQ $4, R10
	24	JS end
	25	JZ one_crc
	26	MOVQ R10, R13
	27	SHRQ $2, R10 // len/4
	28	ANDQ $3, R13 // len&3
	29	XORQ BX, BX
	30	ADDQ $1, R13
	31	TESTQ R10, R10
	32	JZ rem_loop
	33
	34	crc_loop:
	35	MOVQ (R8), R11
	36	XORQ BX, BX
	37	XORQ DX, DX
	38	XORQ DI, DI
	39	MOVQ R11, R12
	40	SHRQ $8, R11
	41	MOVQ R12, AX
	42	MOVQ R11, CX
	43	SHRQ $16, R12
	44	SHRQ $16, R11
	45	MOVQ R12, SI
	46
	47	// CRC32 EAX, EBX
	48	BYTE $0xF2; BYTE $0x0f
	49	BYTE $0x38; BYTE $0xf1; BYTE $0xd8
	50
	51	// CRC32 ECX, EDX
	52	BYTE $0xF2; BYTE $0x0f
	53	BYTE $0x38; BYTE $0xf1; BYTE $0xd1
	54
	55	// CRC32 ESI, EDI
	56	BYTE $0xF2; BYTE $0x0f
	57	BYTE $0x38; BYTE $0xf1; BYTE $0xfe
	58	MOVL BX, (R9)
	59	MOVL DX, 4(R9)
	60	MOVL DI, 8(R9)
	61
	62	XORQ BX, BX
	63	MOVL R11, AX
	64
	65	// CRC32 EAX, EBX
	66	BYTE $0xF2; BYTE $0x0f
	67	BYTE $0x38; BYTE $0xf1; BYTE $0xd8
	68	MOVL BX, 12(R9)
	69
	70	ADDQ $16, R9
	71	ADDQ $4, R8
	72	XORQ BX, BX
	73	SUBQ $1, R10
	74	JNZ crc_loop
	75
	76	rem_loop:
	77	MOVL (R8), AX
	78
	79	// CRC32 EAX, EBX
	80	BYTE $0xF2; BYTE $0x0f
	81	BYTE $0x38; BYTE $0xf1; BYTE $0xd8
	82
	83	MOVL BX, (R9)
	84	ADDQ $4, R9
	85	ADDQ $1, R8
	86	XORQ BX, BX
	87	SUBQ $1, R13
	88	JNZ rem_loop
	89
	90	end:
	91	RET
	92
	93	one_crc:
	94	MOVQ $1, R13
	95	XORQ BX, BX
	96	JMP rem_loop
	97
	98	// func matchLenSSE4(a, b []byte, max int) int
	99	TEXT ·matchLenSSE4(SB), 4, $0
	100	MOVQ a_base+0(FP), SI
	101	MOVQ b_base+24(FP), DI
	102	MOVQ DI, DX
	103	MOVQ max+48(FP), CX
	104
	105	cmp8:
	106	// As long as we are 8 or more bytes before the end of max, we can load and
	107	// compare 8 bytes at a time. If those 8 bytes are equal, repeat.
	108	CMPQ CX, $8
	109	JLT cmp1
	110	MOVQ (SI), AX
	111	MOVQ (DI), BX
	112	CMPQ AX, BX
	113	JNE bsf
	114	ADDQ $8, SI
	115	ADDQ $8, DI
	116	SUBQ $8, CX
	117	JMP cmp8
	118
	119	bsf:
	120	// If those 8 bytes were not equal, XOR the two 8 byte values, and return
	121	// the index of the first byte that differs. The BSF instruction finds the
	122	// least significant 1 bit, the amd64 architecture is little-endian, and
	123	// the shift by 3 converts a bit index to a byte index.
	124	XORQ AX, BX
	125	BSFQ BX, BX
	126	SHRQ $3, BX
	127	ADDQ BX, DI
	128
	129	// Subtract off &b[0] to convert from &b[ret] to ret, and return.
	130	SUBQ DX, DI
	131	MOVQ DI, ret+56(FP)
	132	RET
	133
	134	cmp1:
	135	// In the slices' tail, compare 1 byte at a time.
	136	CMPQ CX, $0
	137	JEQ matchLenEnd
	138	MOVB (SI), AX
	139	MOVB (DI), BX
	140	CMPB AX, BX
	141	JNE matchLenEnd
	142	ADDQ $1, SI
	143	ADDQ $1, DI
	144	SUBQ $1, CX
	145	JMP cmp1
	146
	147	matchLenEnd:
	148	// Subtract off &b[0] to convert from &b[ret] to ret, and return.
	149	SUBQ DX, DI
	150	MOVQ DI, ret+56(FP)
	151	RET
	152
	153	// func histogram(b []byte, h []int32)
	154	TEXT ·histogram(SB), 4, $0
	155	MOVQ b+0(FP), SI // SI: &b
	156	MOVQ b_len+8(FP), R9 // R9: len(b)
	157	MOVQ h+24(FP), DI // DI: Histogram
	158	MOVQ R9, R8
	159	SHRQ $3, R8
	160	JZ hist1
	161	XORQ R11, R11
	162
	163	loop_hist8:
	164	MOVQ (SI), R10
	165
	166	MOVB R10, R11
	167	INCL (DI)(R11*4)
	168	SHRQ $8, R10
	169
	170	MOVB R10, R11
	171	INCL (DI)(R11*4)
	172	SHRQ $8, R10
	173
	174	MOVB R10, R11
	175	INCL (DI)(R11*4)
	176	SHRQ $8, R10
	177
	178	MOVB R10, R11
	179	INCL (DI)(R11*4)
	180	SHRQ $8, R10
	181
	182	MOVB R10, R11
	183	INCL (DI)(R11*4)
	184	SHRQ $8, R10
	185
	186	MOVB R10, R11
	187	INCL (DI)(R11*4)
	188	SHRQ $8, R10
	189
	190	MOVB R10, R11
	191	INCL (DI)(R11*4)
	192	SHRQ $8, R10
	193
	194	INCL (DI)(R10*4)
	195
	196	ADDQ $8, SI
	197	DECQ R8
	198	JNZ loop_hist8
	199
	200	hist1:
	201	ANDQ $7, R9
	202	JZ end_hist
	203	XORQ R10, R10
	204
	205	loop_hist1:
	206	MOVB (SI), R10
	207	INCL (DI)(R10*4)
	208	INCQ SI
	209	DECQ R9
	210	JNZ loop_hist1
	211
	212	end_hist:
	213	RET

vendor/github.com/klauspost/compress/flate/crc32_noasm.go+35−0

@@ -0,0 +1,35 @@
	1	//+build !amd64 noasm appengine
	2
	3	// Copyright 2015, Klaus Post, see LICENSE for details.
	4
	5	package flate
	6
	7	func init() {
	8	useSSE42 = false
	9	}
	10
	11	// crc32sse should never be called.
	12	func crc32sse(a []byte) uint32 {
	13	panic("no assembler")
	14	}
	15
	16	// crc32sseAll should never be called.
	17	func crc32sseAll(a []byte, dst []uint32) {
	18	panic("no assembler")
	19	}
	20
	21	// matchLenSSE4 should never be called.
	22	func matchLenSSE4(a, b []byte, max int) int {
	23	panic("no assembler")
	24	return 0
	25	}
	26
	27	// histogram accumulates a histogram of b in h.
	28	//
	29	// len(h) must be >= 256, and h's elements must be all zeroes.
	30	func histogram(b []byte, h []int32) {
	31	h = h[:256]
	32	for _, t := range b {
	33	h[t]++
	34	}
	35	}

vendor/github.com/klauspost/compress/flate/deflate.go+1353−0

@@ -0,0 +1,1353 @@
	1	// Copyright 2009 The Go Authors. All rights reserved.
	2	// Copyright (c) 2015 Klaus Post
	3	// Use of this source code is governed by a BSD-style
	4	// license that can be found in the LICENSE file.
	5
	6	package flate
	7
	8	import (
	9	"fmt"
	10	"io"
	11	"math"
	12	)
	13
	14	const (
	15	NoCompression = 0
	16	BestSpeed = 1
	17	BestCompression = 9
	18	DefaultCompression = -1
	19
	20	// HuffmanOnly disables Lempel-Ziv match searching and only performs Huffman
	21	// entropy encoding. This mode is useful in compressing data that has
	22	// already been compressed with an LZ style algorithm (e.g. Snappy or LZ4)
	23	// that lacks an entropy encoder. Compression gains are achieved when
	24	// certain bytes in the input stream occur more frequently than others.
	25	//
	26	// Note that HuffmanOnly produces a compressed output that is
	27	// RFC 1951 compliant. That is, any valid DEFLATE decompressor will
	28	// continue to be able to decompress this output.
	29	HuffmanOnly = -2
	30	ConstantCompression = HuffmanOnly // compatibility alias.
	31
	32	logWindowSize = 15
	33	windowSize = 1 << logWindowSize
	34	windowMask = windowSize - 1
	35	logMaxOffsetSize = 15 // Standard DEFLATE
	36	minMatchLength = 4 // The smallest match that the compressor looks for
	37	maxMatchLength = 258 // The longest match for the compressor
	38	minOffsetSize = 1 // The shortest offset that makes any sense
	39
	40	// The maximum number of tokens we put into a single flat block, just too
	41	// stop things from getting too large.
	42	maxFlateBlockTokens = 1 << 14
	43	maxStoreBlockSize = 65535
	44	hashBits = 17 // After 17 performance degrades
	45	hashSize = 1 << hashBits
	46	hashMask = (1 << hashBits) - 1
	47	hashShift = (hashBits + minMatchLength - 1) / minMatchLength
	48	maxHashOffset = 1 << 24
	49
	50	skipNever = math.MaxInt32
	51	)
	52
	53	var useSSE42 bool
	54
	55	type compressionLevel struct {
	56	good, lazy, nice, chain, fastSkipHashing, level int
	57	}
	58
	59	// Compression levels have been rebalanced from zlib deflate defaults
	60	// to give a bigger spread in speed and compression.
	61	// See https://blog.klauspost.com/rebalancing-deflate-compression-levels/
	62	var levels = []compressionLevel{
	63	{}, // 0
	64	// Level 1-4 uses specialized algorithm - values not used
	65	{0, 0, 0, 0, 0, 1},
	66	{0, 0, 0, 0, 0, 2},
	67	{0, 0, 0, 0, 0, 3},
	68	{0, 0, 0, 0, 0, 4},
	69	// For levels 5-6 we don't bother trying with lazy matches.
	70	// Lazy matching is at least 30% slower, with 1.5% increase.
	71	{6, 0, 12, 8, 12, 5},
	72	{8, 0, 24, 16, 16, 6},
	73	// Levels 7-9 use increasingly more lazy matching
	74	// and increasingly stringent conditions for "good enough".
	75	{8, 8, 24, 16, skipNever, 7},
	76	{10, 16, 24, 64, skipNever, 8},
	77	{32, 258, 258, 4096, skipNever, 9},
	78	}
	79
	80	type compressor struct {
	81	compressionLevel
	82
	83	w *huffmanBitWriter
	84	bulkHasher func([]byte, []uint32)
	85
	86	// compression algorithm
	87	fill func(*compressor, []byte) int // copy data to window
	88	step func(*compressor) // process window
	89	sync bool // requesting flush
	90
	91	// Input hash chains
	92	// hashHead[hashValue] contains the largest inputIndex with the specified hash value
	93	// If hashHead[hashValue] is within the current window, then
	94	// hashPrev[hashHead[hashValue] & windowMask] contains the previous index
	95	// with the same hash value.
	96	chainHead int
	97	hashHead [hashSize]uint32
	98	hashPrev [windowSize]uint32
	99	hashOffset int
	100
	101	// input window: unprocessed data is window[index:windowEnd]
	102	index int
	103	window []byte
	104	windowEnd int
	105	blockStart int // window index where current tokens start
	106	byteAvailable bool // if true, still need to process window[index-1].
	107
	108	// queued output tokens
	109	tokens tokens
	110
	111	// deflate state
	112	length int
	113	offset int
	114	hash uint32
	115	maxInsertIndex int
	116	err error
	117	ii uint16 // position of last match, intended to overflow to reset.
	118
	119	snap snappyEnc
	120	hashMatch [maxMatchLength + minMatchLength]uint32
	121	}
	122
	123	func (d *compressor) fillDeflate(b []byte) int {
	124	if d.index >= 2*windowSize-(minMatchLength+maxMatchLength) {
	125	// shift the window by windowSize
	126	copy(d.window[:], d.window[windowSize:2*windowSize])
	127	d.index -= windowSize
	128	d.windowEnd -= windowSize
	129	if d.blockStart >= windowSize {
	130	d.blockStart -= windowSize
	131	} else {
	132	d.blockStart = math.MaxInt32
	133	}
	134	d.hashOffset += windowSize
	135	if d.hashOffset > maxHashOffset {
	136	delta := d.hashOffset - 1
	137	d.hashOffset -= delta
	138	d.chainHead -= delta
	139	// Iterate over slices instead of arrays to avoid copying
	140	// the entire table onto the stack (Issue #18625).
	141	for i, v := range d.hashPrev[:] {
	142	if int(v) > delta {
	143	d.hashPrev[i] = uint32(int(v) - delta)
	144	} else {
	145	d.hashPrev[i] = 0
	146	}
	147	}
	148	for i, v := range d.hashHead[:] {
	149	if int(v) > delta {
	150	d.hashHead[i] = uint32(int(v) - delta)
	151	} else {
	152	d.hashHead[i] = 0
	153	}
	154	}
	155	}
	156	}
	157	n := copy(d.window[d.windowEnd:], b)
	158	d.windowEnd += n
	159	return n
	160	}
	161
	162	func (d *compressor) writeBlock(tok tokens, index int, eof bool) error {
	163	if index > 0 \|\| eof {
	164	var window []byte
	165	if d.blockStart <= index {
	166	window = d.window[d.blockStart:index]
	167	}
	168	d.blockStart = index
	169	d.w.writeBlock(tok.tokens[:tok.n], eof, window)
	170	return d.w.err
	171	}
	172	return nil
	173	}
	174
	175	// writeBlockSkip writes the current block and uses the number of tokens
	176	// to determine if the block should be stored on no matches, or
	177	// only huffman encoded.
	178	func (d *compressor) writeBlockSkip(tok tokens, index int, eof bool) error {
	179	if index > 0 \|\| eof {
	180	if d.blockStart <= index {
	181	window := d.window[d.blockStart:index]
	182	// If we removed less than a 64th of all literals
	183	// we huffman compress the block.
	184	if int(tok.n) > len(window)-int(tok.n>>6) {
	185	d.w.writeBlockHuff(eof, window)
	186	} else {
	187	// Write a dynamic huffman block.
	188	d.w.writeBlockDynamic(tok.tokens[:tok.n], eof, window)
	189	}
	190	} else {
	191	d.w.writeBlock(tok.tokens[:tok.n], eof, nil)
	192	}
	193	d.blockStart = index
	194	return d.w.err
	195	}
	196	return nil
	197	}
	198
	199	// fillWindow will fill the current window with the supplied
	200	// dictionary and calculate all hashes.
	201	// This is much faster than doing a full encode.
	202	// Should only be used after a start/reset.
	203	func (d *compressor) fillWindow(b []byte) {
	204	// Do not fill window if we are in store-only mode,
	205	// use constant or Snappy compression.
	206	switch d.compressionLevel.level {
	207	case 0, 1, 2:
	208	return
	209	}
	210	// If we are given too much, cut it.
	211	if len(b) > windowSize {
	212	b = b[len(b)-windowSize:]
	213	}
	214	// Add all to window.
	215	n := copy(d.window[d.windowEnd:], b)
	216
	217	// Calculate 256 hashes at the time (more L1 cache hits)
	218	loops := (n + 256 - minMatchLength) / 256
	219	for j := 0; j < loops; j++ {
	220	startindex := j * 256
	221	end := startindex + 256 + minMatchLength - 1
	222	if end > n {
	223	end = n
	224	}
	225	tocheck := d.window[startindex:end]
	226	dstSize := len(tocheck) - minMatchLength + 1
	227
	228	if dstSize <= 0 {
	229	continue
	230	}
	231
	232	dst := d.hashMatch[:dstSize]
	233	d.bulkHasher(tocheck, dst)
	234	var newH uint32
	235	for i, val := range dst {
	236	di := i + startindex
	237	newH = val & hashMask
	238	// Get previous value with the same hash.
	239	// Our chain should point to the previous value.
	240	d.hashPrev[di&windowMask] = d.hashHead[newH]
	241	// Set the head of the hash chain to us.
	242	d.hashHead[newH] = uint32(di + d.hashOffset)
	243	}
	244	d.hash = newH
	245	}
	246	// Update window information.
	247	d.windowEnd += n
	248	d.index = n
	249	}
	250
	251	// Try to find a match starting at index whose length is greater than prevSize.
	252	// We only look at chainCount possibilities before giving up.
	253	// pos = d.index, prevHead = d.chainHead-d.hashOffset, prevLength=minMatchLength-1, lookahead
	254	func (d *compressor) findMatch(pos int, prevHead int, prevLength int, lookahead int) (length, offset int, ok bool) {
	255	minMatchLook := maxMatchLength
	256	if lookahead < minMatchLook {
	257	minMatchLook = lookahead
	258	}
	259
	260	win := d.window[0 : pos+minMatchLook]
	261
	262	// We quit when we get a match that's at least nice long
	263	nice := len(win) - pos
	264	if d.nice < nice {
	265	nice = d.nice
	266	}
	267
	268	// If we've got a match that's good enough, only look in 1/4 the chain.
	269	tries := d.chain
	270	length = prevLength
	271	if length >= d.good {
	272	tries >>= 2
	273	}
	274
	275	wEnd := win[pos+length]
	276	wPos := win[pos:]
	277	minIndex := pos - windowSize
	278
	279	for i := prevHead; tries > 0; tries-- {
	280	if wEnd == win[i+length] {
	281	n := matchLen(win[i:], wPos, minMatchLook)
	282
	283	if n > length && (n > minMatchLength \|\| pos-i <= 4096) {
	284	length = n
	285	offset = pos - i
	286	ok = true
	287	if n >= nice {
	288	// The match is good enough that we don't try to find a better one.
	289	break
	290	}
	291	wEnd = win[pos+n]
	292	}
	293	}
	294	if i == minIndex {
	295	// hashPrev[i & windowMask] has already been overwritten, so stop now.
	296	break
	297	}
	298	i = int(d.hashPrev[i&windowMask]) - d.hashOffset
	299	if i < minIndex \|\| i < 0 {
	300	break
	301	}
	302	}
	303	return
	304	}
	305
	306	// Try to find a match starting at index whose length is greater than prevSize.
	307	// We only look at chainCount possibilities before giving up.
	308	// pos = d.index, prevHead = d.chainHead-d.hashOffset, prevLength=minMatchLength-1, lookahead
	309	func (d *compressor) findMatchSSE(pos int, prevHead int, prevLength int, lookahead int) (length, offset int, ok bool) {
	310	minMatchLook := maxMatchLength
	311	if lookahead < minMatchLook {
	312	minMatchLook = lookahead
	313	}
	314
	315	win := d.window[0 : pos+minMatchLook]
	316
	317	// We quit when we get a match that's at least nice long
	318	nice := len(win) - pos
	319	if d.nice < nice {
	320	nice = d.nice
	321	}
	322
	323	// If we've got a match that's good enough, only look in 1/4 the chain.
	324	tries := d.chain
	325	length = prevLength
	326	if length >= d.good {
	327	tries >>= 2
	328	}
	329
	330	wEnd := win[pos+length]
	331	wPos := win[pos:]
	332	minIndex := pos - windowSize
	333
	334	for i := prevHead; tries > 0; tries-- {
	335	if wEnd == win[i+length] {
	336	n := matchLenSSE4(win[i:], wPos, minMatchLook)
	337
	338	if n > length && (n > minMatchLength \|\| pos-i <= 4096) {
	339	length = n
	340	offset = pos - i
	341	ok = true
	342	if n >= nice {
	343	// The match is good enough that we don't try to find a better one.
	344	break
	345	}
	346	wEnd = win[pos+n]
	347	}
	348	}
	349	if i == minIndex {
	350	// hashPrev[i & windowMask] has already been overwritten, so stop now.
	351	break
	352	}
	353	i = int(d.hashPrev[i&windowMask]) - d.hashOffset
	354	if i < minIndex \|\| i < 0 {
	355	break
	356	}
	357	}
	358	return
	359	}
	360
	361	func (d *compressor) writeStoredBlock(buf []byte) error {
	362	if d.w.writeStoredHeader(len(buf), false); d.w.err != nil {
	363	return d.w.err
	364	}
	365	d.w.writeBytes(buf)
	366	return d.w.err
	367	}
	368
	369	const hashmul = 0x1e35a7bd
	370
	371	// hash4 returns a hash representation of the first 4 bytes
	372	// of the supplied slice.
	373	// The caller must ensure that len(b) >= 4.
	374	func hash4(b []byte) uint32 {
	375	return ((uint32(b[3]) \| uint32(b[2])<<8 \| uint32(b[1])<<16 \| uint32(b[0])<<24) * hashmul) >> (32 - hashBits)
	376	}
	377
	378	// bulkHash4 will compute hashes using the same
	379	// algorithm as hash4
	380	func bulkHash4(b []byte, dst []uint32) {
	381	if len(b) < minMatchLength {
	382	return
	383	}
	384	hb := uint32(b[3]) \| uint32(b[2])<<8 \| uint32(b[1])<<16 \| uint32(b[0])<<24
	385	dst[0] = (hb * hashmul) >> (32 - hashBits)
	386	end := len(b) - minMatchLength + 1
	387	for i := 1; i < end; i++ {
	388	hb = (hb << 8) \| uint32(b[i+3])
	389	dst[i] = (hb * hashmul) >> (32 - hashBits)
	390	}
	391	}
	392
	393	// matchLen returns the number of matching bytes in a and b
	394	// up to length 'max'. Both slices must be at least 'max'
	395	// bytes in size.
	396	func matchLen(a, b []byte, max int) int {
	397	a = a[:max]
	398	b = b[:len(a)]
	399	for i, av := range a {
	400	if b[i] != av {
	401	return i
	402	}
	403	}
	404	return max
	405	}
	406
	407	func (d *compressor) initDeflate() {
	408	d.window = make([]byte, 2*windowSize)
	409	d.hashOffset = 1
	410	d.length = minMatchLength - 1
	411	d.offset = 0
	412	d.byteAvailable = false
	413	d.index = 0
	414	d.hash = 0
	415	d.chainHead = -1
	416	d.bulkHasher = bulkHash4
	417	if useSSE42 {
	418	d.bulkHasher = crc32sseAll
	419	}
	420	}
	421
	422	// Assumes that d.fastSkipHashing != skipNever,
	423	// otherwise use deflateLazy
	424	func (d *compressor) deflate() {
	425
	426	// Sanity enables additional runtime tests.
	427	// It's intended to be used during development
	428	// to supplement the currently ad-hoc unit tests.
	429	const sanity = false
	430
	431	if d.windowEnd-d.index < minMatchLength+maxMatchLength && !d.sync {
	432	return
	433	}
	434
	435	d.maxInsertIndex = d.windowEnd - (minMatchLength - 1)
	436	if d.index < d.maxInsertIndex {
	437	d.hash = hash4(d.window[d.index : d.index+minMatchLength])
	438	}
	439
	440	for {
	441	if sanity && d.index > d.windowEnd {
	442	panic("index > windowEnd")
	443	}
	444	lookahead := d.windowEnd - d.index
	445	if lookahead < minMatchLength+maxMatchLength {
	446	if !d.sync {
	447	return
	448	}
	449	if sanity && d.index > d.windowEnd {
	450	panic("index > windowEnd")
	451	}
	452	if lookahead == 0 {
	453	if d.tokens.n > 0 {
	454	if d.err = d.writeBlockSkip(d.tokens, d.index, false); d.err != nil {
	455	return
	456	}
	457	d.tokens.n = 0
	458	}
	459	return
	460	}
	461	}
	462	if d.index < d.maxInsertIndex {
	463	// Update the hash
	464	d.hash = hash4(d.window[d.index : d.index+minMatchLength])
	465	ch := d.hashHead[d.hash&hashMask]
	466	d.chainHead = int(ch)
	467	d.hashPrev[d.index&windowMask] = ch
	468	d.hashHead[d.hash&hashMask] = uint32(d.index + d.hashOffset)
	469	}
	470	d.length = minMatchLength - 1
	471	d.offset = 0
	472	minIndex := d.index - windowSize
	473	if minIndex < 0 {
	474	minIndex = 0
	475	}
	476
	477	if d.chainHead-d.hashOffset >= minIndex && lookahead > minMatchLength-1 {
	478	if newLength, newOffset, ok := d.findMatch(d.index, d.chainHead-d.hashOffset, minMatchLength-1, lookahead); ok {
	479	d.length = newLength
	480	d.offset = newOffset
	481	}
	482	}
	483	if d.length >= minMatchLength {
	484	d.ii = 0
	485	// There was a match at the previous step, and the current match is
	486	// not better. Output the previous match.
	487	// "d.length-3" should NOT be "d.length-minMatchLength", since the format always assume 3
	488	d.tokens.tokens[d.tokens.n] = matchToken(uint32(d.length-3), uint32(d.offset-minOffsetSize))
	489	d.tokens.n++
	490	// Insert in the hash table all strings up to the end of the match.
	491	// index and index-1 are already inserted. If there is not enough
	492	// lookahead, the last two strings are not inserted into the hash
	493	// table.
	494	if d.length <= d.fastSkipHashing {
	495	var newIndex int
	496	newIndex = d.index + d.length
	497	// Calculate missing hashes
	498	end := newIndex
	499	if end > d.maxInsertIndex {
	500	end = d.maxInsertIndex
	501	}
	502	end += minMatchLength - 1
	503	startindex := d.index + 1
	504	if startindex > d.maxInsertIndex {
	505	startindex = d.maxInsertIndex
	506	}
	507	tocheck := d.window[startindex:end]
	508	dstSize := len(tocheck) - minMatchLength + 1
	509	if dstSize > 0 {
	510	dst := d.hashMatch[:dstSize]
	511	bulkHash4(tocheck, dst)
	512	var newH uint32
	513	for i, val := range dst {
	514	di := i + startindex
	515	newH = val & hashMask
	516	// Get previous value with the same hash.
	517	// Our chain should point to the previous value.
	518	d.hashPrev[di&windowMask] = d.hashHead[newH]
	519	// Set the head of the hash chain to us.
	520	d.hashHead[newH] = uint32(di + d.hashOffset)
	521	}
	522	d.hash = newH
	523	}
	524	d.index = newIndex
	525	} else {
	526	// For matches this long, we don't bother inserting each individual
	527	// item into the table.
	528	d.index += d.length
	529	if d.index < d.maxInsertIndex {
	530	d.hash = hash4(d.window[d.index : d.index+minMatchLength])
	531	}
	532	}
	533	if d.tokens.n == maxFlateBlockTokens {
	534	// The block includes the current character
	535	if d.err = d.writeBlockSkip(d.tokens, d.index, false); d.err != nil {
	536	return
	537	}
	538	d.tokens.n = 0
	539	}
	540	} else {
	541	d.ii++
	542	end := d.index + int(d.ii>>uint(d.fastSkipHashing)) + 1
	543	if end > d.windowEnd {
	544	end = d.windowEnd
	545	}
	546	for i := d.index; i < end; i++ {
	547	d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[i]))
	548	d.tokens.n++
	549	if d.tokens.n == maxFlateBlockTokens {
	550	if d.err = d.writeBlockSkip(d.tokens, i+1, false); d.err != nil {
	551	return
	552	}
	553	d.tokens.n = 0
	554	}
	555	}
	556	d.index = end
	557	}
	558	}
	559	}
	560
	561	// deflateLazy is the same as deflate, but with d.fastSkipHashing == skipNever,
	562	// meaning it always has lazy matching on.
	563	func (d *compressor) deflateLazy() {
	564	// Sanity enables additional runtime tests.
	565	// It's intended to be used during development
	566	// to supplement the currently ad-hoc unit tests.
	567	const sanity = false
	568
	569	if d.windowEnd-d.index < minMatchLength+maxMatchLength && !d.sync {
	570	return
	571	}
	572
	573	d.maxInsertIndex = d.windowEnd - (minMatchLength - 1)
	574	if d.index < d.maxInsertIndex {
	575	d.hash = hash4(d.window[d.index : d.index+minMatchLength])
	576	}
	577
	578	for {
	579	if sanity && d.index > d.windowEnd {
	580	panic("index > windowEnd")
	581	}
	582	lookahead := d.windowEnd - d.index
	583	if lookahead < minMatchLength+maxMatchLength {
	584	if !d.sync {
	585	return
	586	}
	587	if sanity && d.index > d.windowEnd {
	588	panic("index > windowEnd")
	589	}
	590	if lookahead == 0 {
	591	// Flush current output block if any.
	592	if d.byteAvailable {
	593	// There is still one pending token that needs to be flushed
	594	d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1]))
	595	d.tokens.n++
	596	d.byteAvailable = false
	597	}
	598	if d.tokens.n > 0 {
	599	if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil {
	600	return
	601	}
	602	d.tokens.n = 0
	603	}
	604	return
	605	}
	606	}
	607	if d.index < d.maxInsertIndex {
	608	// Update the hash
	609	d.hash = hash4(d.window[d.index : d.index+minMatchLength])
	610	ch := d.hashHead[d.hash&hashMask]
	611	d.chainHead = int(ch)
	612	d.hashPrev[d.index&windowMask] = ch
	613	d.hashHead[d.hash&hashMask] = uint32(d.index + d.hashOffset)
	614	}
	615	prevLength := d.length
	616	prevOffset := d.offset
	617	d.length = minMatchLength - 1
	618	d.offset = 0
	619	minIndex := d.index - windowSize
	620	if minIndex < 0 {
	621	minIndex = 0
	622	}
	623
	624	if d.chainHead-d.hashOffset >= minIndex && lookahead > prevLength && prevLength < d.lazy {
	625	if newLength, newOffset, ok := d.findMatch(d.index, d.chainHead-d.hashOffset, minMatchLength-1, lookahead); ok {
	626	d.length = newLength
	627	d.offset = newOffset
	628	}
	629	}
	630	if prevLength >= minMatchLength && d.length <= prevLength {
	631	// There was a match at the previous step, and the current match is
	632	// not better. Output the previous match.
	633	d.tokens.tokens[d.tokens.n] = matchToken(uint32(prevLength-3), uint32(prevOffset-minOffsetSize))
	634	d.tokens.n++
	635
	636	// Insert in the hash table all strings up to the end of the match.
	637	// index and index-1 are already inserted. If there is not enough
	638	// lookahead, the last two strings are not inserted into the hash
	639	// table.
	640	var newIndex int
	641	newIndex = d.index + prevLength - 1
	642	// Calculate missing hashes
	643	end := newIndex
	644	if end > d.maxInsertIndex {
	645	end = d.maxInsertIndex
	646	}
	647	end += minMatchLength - 1
	648	startindex := d.index + 1
	649	if startindex > d.maxInsertIndex {
	650	startindex = d.maxInsertIndex
	651	}
	652	tocheck := d.window[startindex:end]
	653	dstSize := len(tocheck) - minMatchLength + 1
	654	if dstSize > 0 {
	655	dst := d.hashMatch[:dstSize]
	656	bulkHash4(tocheck, dst)
	657	var newH uint32
	658	for i, val := range dst {
	659	di := i + startindex
	660	newH = val & hashMask
	661	// Get previous value with the same hash.
	662	// Our chain should point to the previous value.
	663	d.hashPrev[di&windowMask] = d.hashHead[newH]
	664	// Set the head of the hash chain to us.
	665	d.hashHead[newH] = uint32(di + d.hashOffset)
	666	}
	667	d.hash = newH
	668	}
	669
	670	d.index = newIndex
	671	d.byteAvailable = false
	672	d.length = minMatchLength - 1
	673	if d.tokens.n == maxFlateBlockTokens {
	674	// The block includes the current character
	675	if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil {
	676	return
	677	}
	678	d.tokens.n = 0
	679	}
	680	} else {
	681	// Reset, if we got a match this run.
	682	if d.length >= minMatchLength {
	683	d.ii = 0
	684	}
	685	// We have a byte waiting. Emit it.
	686	if d.byteAvailable {
	687	d.ii++
	688	d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1]))
	689	d.tokens.n++
	690	if d.tokens.n == maxFlateBlockTokens {
	691	if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil {
	692	return
	693	}
	694	d.tokens.n = 0
	695	}
	696	d.index++
	697
	698	// If we have a long run of no matches, skip additional bytes
	699	// Resets when d.ii overflows after 64KB.
	700	if d.ii > 31 {
	701	n := int(d.ii >> 5)
	702	for j := 0; j < n; j++ {
	703	if d.index >= d.windowEnd-1 {
	704	break
	705	}
	706
	707	d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1]))
	708	d.tokens.n++
	709	if d.tokens.n == maxFlateBlockTokens {
	710	if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil {
	711	return
	712	}
	713	d.tokens.n = 0
	714	}
	715	d.index++
	716	}
	717	// Flush last byte
	718	d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1]))
	719	d.tokens.n++
	720	d.byteAvailable = false
	721	// d.length = minMatchLength - 1 // not needed, since d.ii is reset above, so it should never be > minMatchLength
	722	if d.tokens.n == maxFlateBlockTokens {
	723	if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil {
	724	return
	725	}
	726	d.tokens.n = 0
	727	}
	728	}
	729	} else {
	730	d.index++
	731	d.byteAvailable = true
	732	}
	733	}
	734	}
	735	}
	736
	737	// Assumes that d.fastSkipHashing != skipNever,
	738	// otherwise use deflateLazySSE
	739	func (d *compressor) deflateSSE() {
	740
	741	// Sanity enables additional runtime tests.
	742	// It's intended to be used during development
	743	// to supplement the currently ad-hoc unit tests.
	744	const sanity = false
	745
	746	if d.windowEnd-d.index < minMatchLength+maxMatchLength && !d.sync {
	747	return
	748	}
	749
	750	d.maxInsertIndex = d.windowEnd - (minMatchLength - 1)
	751	if d.index < d.maxInsertIndex {
	752	d.hash = crc32sse(d.window[d.index:d.index+minMatchLength]) & hashMask
	753	}
	754
	755	for {
	756	if sanity && d.index > d.windowEnd {
	757	panic("index > windowEnd")
	758	}
	759	lookahead := d.windowEnd - d.index
	760	if lookahead < minMatchLength+maxMatchLength {
	761	if !d.sync {
	762	return
	763	}
	764	if sanity && d.index > d.windowEnd {
	765	panic("index > windowEnd")
	766	}
	767	if lookahead == 0 {
	768	if d.tokens.n > 0 {
	769	if d.err = d.writeBlockSkip(d.tokens, d.index, false); d.err != nil {
	770	return
	771	}
	772	d.tokens.n = 0
	773	}
	774	return
	775	}
	776	}
	777	if d.index < d.maxInsertIndex {
	778	// Update the hash
	779	d.hash = crc32sse(d.window[d.index:d.index+minMatchLength]) & hashMask
	780	ch := d.hashHead[d.hash]
	781	d.chainHead = int(ch)
	782	d.hashPrev[d.index&windowMask] = ch
	783	d.hashHead[d.hash] = uint32(d.index + d.hashOffset)
	784	}
	785	d.length = minMatchLength - 1
	786	d.offset = 0
	787	minIndex := d.index - windowSize
	788	if minIndex < 0 {
	789	minIndex = 0
	790	}
	791
	792	if d.chainHead-d.hashOffset >= minIndex && lookahead > minMatchLength-1 {
	793	if newLength, newOffset, ok := d.findMatchSSE(d.index, d.chainHead-d.hashOffset, minMatchLength-1, lookahead); ok {
	794	d.length = newLength
	795	d.offset = newOffset
	796	}
	797	}
	798	if d.length >= minMatchLength {
	799	d.ii = 0
	800	// There was a match at the previous step, and the current match is
	801	// not better. Output the previous match.
	802	// "d.length-3" should NOT be "d.length-minMatchLength", since the format always assume 3
	803	d.tokens.tokens[d.tokens.n] = matchToken(uint32(d.length-3), uint32(d.offset-minOffsetSize))
	804	d.tokens.n++
	805	// Insert in the hash table all strings up to the end of the match.
	806	// index and index-1 are already inserted. If there is not enough
	807	// lookahead, the last two strings are not inserted into the hash
	808	// table.
	809	if d.length <= d.fastSkipHashing {
	810	var newIndex int
	811	newIndex = d.index + d.length
	812	// Calculate missing hashes
	813	end := newIndex
	814	if end > d.maxInsertIndex {
	815	end = d.maxInsertIndex
	816	}
	817	end += minMatchLength - 1
	818	startindex := d.index + 1
	819	if startindex > d.maxInsertIndex {
	820	startindex = d.maxInsertIndex
	821	}
	822	tocheck := d.window[startindex:end]
	823	dstSize := len(tocheck) - minMatchLength + 1
	824	if dstSize > 0 {
	825	dst := d.hashMatch[:dstSize]
	826
	827	crc32sseAll(tocheck, dst)
	828	var newH uint32
	829	for i, val := range dst {
	830	di := i + startindex
	831	newH = val & hashMask
	832	// Get previous value with the same hash.
	833	// Our chain should point to the previous value.
	834	d.hashPrev[di&windowMask] = d.hashHead[newH]
	835	// Set the head of the hash chain to us.
	836	d.hashHead[newH] = uint32(di + d.hashOffset)
	837	}
	838	d.hash = newH
	839	}
	840	d.index = newIndex
	841	} else {
	842	// For matches this long, we don't bother inserting each individual
	843	// item into the table.
	844	d.index += d.length
	845	if d.index < d.maxInsertIndex {
	846	d.hash = crc32sse(d.window[d.index:d.index+minMatchLength]) & hashMask
	847	}
	848	}
	849	if d.tokens.n == maxFlateBlockTokens {
	850	// The block includes the current character
	851	if d.err = d.writeBlockSkip(d.tokens, d.index, false); d.err != nil {
	852	return
	853	}
	854	d.tokens.n = 0
	855	}
	856	} else {
	857	d.ii++
	858	end := d.index + int(d.ii>>5) + 1
	859	if end > d.windowEnd {
	860	end = d.windowEnd
	861	}
	862	for i := d.index; i < end; i++ {
	863	d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[i]))
	864	d.tokens.n++
	865	if d.tokens.n == maxFlateBlockTokens {
	866	if d.err = d.writeBlockSkip(d.tokens, i+1, false); d.err != nil {
	867	return
	868	}
	869	d.tokens.n = 0
	870	}
	871	}
	872	d.index = end
	873	}
	874	}
	875	}
	876
	877	// deflateLazy is the same as deflate, but with d.fastSkipHashing == skipNever,
	878	// meaning it always has lazy matching on.
	879	func (d *compressor) deflateLazySSE() {
	880	// Sanity enables additional runtime tests.
	881	// It's intended to be used during development
	882	// to supplement the currently ad-hoc unit tests.
	883	const sanity = false
	884
	885	if d.windowEnd-d.index < minMatchLength+maxMatchLength && !d.sync {
	886	return
	887	}
	888
	889	d.maxInsertIndex = d.windowEnd - (minMatchLength - 1)
	890	if d.index < d.maxInsertIndex {
	891	d.hash = crc32sse(d.window[d.index:d.index+minMatchLength]) & hashMask
	892	}
	893
	894	for {
	895	if sanity && d.index > d.windowEnd {
	896	panic("index > windowEnd")
	897	}
	898	lookahead := d.windowEnd - d.index
	899	if lookahead < minMatchLength+maxMatchLength {
	900	if !d.sync {
	901	return
	902	}
	903	if sanity && d.index > d.windowEnd {
	904	panic("index > windowEnd")
	905	}
	906	if lookahead == 0 {
	907	// Flush current output block if any.
	908	if d.byteAvailable {
	909	// There is still one pending token that needs to be flushed
	910	d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1]))
	911	d.tokens.n++
	912	d.byteAvailable = false
	913	}
	914	if d.tokens.n > 0 {
	915	if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil {
	916	return
	917	}
	918	d.tokens.n = 0
	919	}
	920	return
	921	}
	922	}
	923	if d.index < d.maxInsertIndex {
	924	// Update the hash
	925	d.hash = crc32sse(d.window[d.index:d.index+minMatchLength]) & hashMask
	926	ch := d.hashHead[d.hash]
	927	d.chainHead = int(ch)
	928	d.hashPrev[d.index&windowMask] = ch
	929	d.hashHead[d.hash] = uint32(d.index + d.hashOffset)
	930	}
	931	prevLength := d.length
	932	prevOffset := d.offset
	933	d.length = minMatchLength - 1
	934	d.offset = 0
	935	minIndex := d.index - windowSize
	936	if minIndex < 0 {
	937	minIndex = 0
	938	}
	939
	940	if d.chainHead-d.hashOffset >= minIndex && lookahead > prevLength && prevLength < d.lazy {
	941	if newLength, newOffset, ok := d.findMatchSSE(d.index, d.chainHead-d.hashOffset, minMatchLength-1, lookahead); ok {
	942	d.length = newLength
	943	d.offset = newOffset
	944	}
	945	}
	946	if prevLength >= minMatchLength && d.length <= prevLength {
	947	// There was a match at the previous step, and the current match is
	948	// not better. Output the previous match.
	949	d.tokens.tokens[d.tokens.n] = matchToken(uint32(prevLength-3), uint32(prevOffset-minOffsetSize))
	950	d.tokens.n++
	951
	952	// Insert in the hash table all strings up to the end of the match.
	953	// index and index-1 are already inserted. If there is not enough
	954	// lookahead, the last two strings are not inserted into the hash
	955	// table.
	956	var newIndex int
	957	newIndex = d.index + prevLength - 1
	958	// Calculate missing hashes
	959	end := newIndex
	960	if end > d.maxInsertIndex {
	961	end = d.maxInsertIndex
	962	}
	963	end += minMatchLength - 1
	964	startindex := d.index + 1
	965	if startindex > d.maxInsertIndex {
	966	startindex = d.maxInsertIndex
	967	}
	968	tocheck := d.window[startindex:end]
	969	dstSize := len(tocheck) - minMatchLength + 1
	970	if dstSize > 0 {
	971	dst := d.hashMatch[:dstSize]
	972	crc32sseAll(tocheck, dst)
	973	var newH uint32
	974	for i, val := range dst {
	975	di := i + startindex
	976	newH = val & hashMask
	977	// Get previous value with the same hash.
	978	// Our chain should point to the previous value.
	979	d.hashPrev[di&windowMask] = d.hashHead[newH]
	980	// Set the head of the hash chain to us.
	981	d.hashHead[newH] = uint32(di + d.hashOffset)
	982	}
	983	d.hash = newH
	984	}
	985
	986	d.index = newIndex
	987	d.byteAvailable = false
	988	d.length = minMatchLength - 1
	989	if d.tokens.n == maxFlateBlockTokens {
	990	// The block includes the current character
	991	if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil {
	992	return
	993	}
	994	d.tokens.n = 0
	995	}
	996	} else {
	997	// Reset, if we got a match this run.
	998	if d.length >= minMatchLength {
	999	d.ii = 0
	1000	}
	1001	// We have a byte waiting. Emit it.
	1002	if d.byteAvailable {
	1003	d.ii++
	1004	d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1]))
	1005	d.tokens.n++
	1006	if d.tokens.n == maxFlateBlockTokens {
	1007	if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil {
	1008	return
	1009	}
	1010	d.tokens.n = 0
	1011	}
	1012	d.index++
	1013
	1014	// If we have a long run of no matches, skip additional bytes
	1015	// Resets when d.ii overflows after 64KB.
	1016	if d.ii > 31 {
	1017	n := int(d.ii >> 6)
	1018	for j := 0; j < n; j++ {
	1019	if d.index >= d.windowEnd-1 {
	1020	break
	1021	}
	1022
	1023	d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1]))
	1024	d.tokens.n++
	1025	if d.tokens.n == maxFlateBlockTokens {
	1026	if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil {
	1027	return
	1028	}
	1029	d.tokens.n = 0
	1030	}
	1031	d.index++
	1032	}
	1033	// Flush last byte
	1034	d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1]))
	1035	d.tokens.n++
	1036	d.byteAvailable = false
	1037	// d.length = minMatchLength - 1 // not needed, since d.ii is reset above, so it should never be > minMatchLength
	1038	if d.tokens.n == maxFlateBlockTokens {
	1039	if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil {
	1040	return
	1041	}
	1042	d.tokens.n = 0
	1043	}
	1044	}
	1045	} else {
	1046	d.index++
	1047	d.byteAvailable = true
	1048	}
	1049	}
	1050	}
	1051	}
	1052
	1053	func (d *compressor) store() {
	1054	if d.windowEnd > 0 && (d.windowEnd == maxStoreBlockSize \|\| d.sync) {
	1055	d.err = d.writeStoredBlock(d.window[:d.windowEnd])
	1056	d.windowEnd = 0
	1057	}
	1058	}
	1059
	1060	// fillWindow will fill the buffer with data for huffman-only compression.
	1061	// The number of bytes copied is returned.
	1062	func (d *compressor) fillBlock(b []byte) int {
	1063	n := copy(d.window[d.windowEnd:], b)
	1064	d.windowEnd += n
	1065	return n
	1066	}
	1067
	1068	// storeHuff will compress and store the currently added data,
	1069	// if enough has been accumulated or we at the end of the stream.
	1070	// Any error that occurred will be in d.err
	1071	func (d *compressor) storeHuff() {
	1072	if d.windowEnd < len(d.window) && !d.sync \|\| d.windowEnd == 0 {
	1073	return
	1074	}
	1075	d.w.writeBlockHuff(false, d.window[:d.windowEnd])
	1076	d.err = d.w.err
	1077	d.windowEnd = 0
	1078	}
	1079
	1080	// storeHuff will compress and store the currently added data,
	1081	// if enough has been accumulated or we at the end of the stream.
	1082	// Any error that occurred will be in d.err
	1083	func (d *compressor) storeSnappy() {
	1084	// We only compress if we have maxStoreBlockSize.
	1085	if d.windowEnd < maxStoreBlockSize {
	1086	if !d.sync {
	1087	return
	1088	}
	1089	// Handle extremely small sizes.
	1090	if d.windowEnd < 128 {
	1091	if d.windowEnd == 0 {
	1092	return
	1093	}
	1094	if d.windowEnd <= 32 {
	1095	d.err = d.writeStoredBlock(d.window[:d.windowEnd])
	1096	d.tokens.n = 0
	1097	d.windowEnd = 0
	1098	} else {
	1099	d.w.writeBlockHuff(false, d.window[:d.windowEnd])
	1100	d.err = d.w.err
	1101	}
	1102	d.tokens.n = 0
	1103	d.windowEnd = 0
	1104	d.snap.Reset()
	1105	return
	1106	}
	1107	}
	1108
	1109	d.snap.Encode(&d.tokens, d.window[:d.windowEnd])
	1110	// If we made zero matches, store the block as is.
	1111	if int(d.tokens.n) == d.windowEnd {
	1112	d.err = d.writeStoredBlock(d.window[:d.windowEnd])
	1113	// If we removed less than 1/16th, huffman compress the block.
	1114	} else if int(d.tokens.n) > d.windowEnd-(d.windowEnd>>4) {
	1115	d.w.writeBlockHuff(false, d.window[:d.windowEnd])
	1116	d.err = d.w.err
	1117	} else {
	1118	d.w.writeBlockDynamic(d.tokens.tokens[:d.tokens.n], false, d.window[:d.windowEnd])
	1119	d.err = d.w.err
	1120	}
	1121	d.tokens.n = 0
	1122	d.windowEnd = 0
	1123	}
	1124
	1125	// write will add input byte to the stream.
	1126	// Unless an error occurs all bytes will be consumed.
	1127	func (d *compressor) write(b []byte) (n int, err error) {
	1128	if d.err != nil {
	1129	return 0, d.err
	1130	}
	1131	n = len(b)
	1132	for len(b) > 0 {
	1133	d.step(d)
	1134	b = b[d.fill(d, b):]
	1135	if d.err != nil {
	1136	return 0, d.err
	1137	}
	1138	}
	1139	return n, d.err
	1140	}
	1141
	1142	func (d *compressor) syncFlush() error {
	1143	d.sync = true
	1144	if d.err != nil {
	1145	return d.err
	1146	}
	1147	d.step(d)
	1148	if d.err == nil {
	1149	d.w.writeStoredHeader(0, false)
	1150	d.w.flush()
	1151	d.err = d.w.err
	1152	}
	1153	d.sync = false
	1154	return d.err
	1155	}
	1156
	1157	func (d *compressor) init(w io.Writer, level int) (err error) {
	1158	d.w = newHuffmanBitWriter(w)
	1159
	1160	switch {
	1161	case level == NoCompression:
	1162	d.window = make([]byte, maxStoreBlockSize)
	1163	d.fill = (*compressor).fillBlock
	1164	d.step = (*compressor).store
	1165	case level == ConstantCompression:
	1166	d.window = make([]byte, maxStoreBlockSize)
	1167	d.fill = (*compressor).fillBlock
	1168	d.step = (*compressor).storeHuff
	1169	case level >= 1 && level <= 4:
	1170	d.snap = newSnappy(level)
	1171	d.window = make([]byte, maxStoreBlockSize)
	1172	d.fill = (*compressor).fillBlock
	1173	d.step = (*compressor).storeSnappy
	1174	case level == DefaultCompression:
	1175	level = 5
	1176	fallthrough
	1177	case 5 <= level && level <= 9:
	1178	d.compressionLevel = levels[level]
	1179	d.initDeflate()
	1180	d.fill = (*compressor).fillDeflate
	1181	if d.fastSkipHashing == skipNever {
	1182	if useSSE42 {
	1183	d.step = (*compressor).deflateLazySSE
	1184	} else {
	1185	d.step = (*compressor).deflateLazy
	1186	}
	1187	} else {
	1188	if useSSE42 {
	1189	d.step = (*compressor).deflateSSE
	1190	} else {
	1191	d.step = (*compressor).deflate
	1192
	1193	}
	1194	}
	1195	default:
	1196	return fmt.Errorf("flate: invalid compression level %d: want value in range [-2, 9]", level)
	1197	}
	1198	return nil
	1199	}
	1200
	1201	// reset the state of the compressor.
	1202	func (d *compressor) reset(w io.Writer) {
	1203	d.w.reset(w)
	1204	d.sync = false
	1205	d.err = nil
	1206	// We only need to reset a few things for Snappy.
	1207	if d.snap != nil {
	1208	d.snap.Reset()
	1209	d.windowEnd = 0
	1210	d.tokens.n = 0
	1211	return
	1212	}
	1213	switch d.compressionLevel.chain {
	1214	case 0:
	1215	// level was NoCompression or ConstantCompresssion.
	1216	d.windowEnd = 0
	1217	default:
	1218	d.chainHead = -1
	1219	for i := range d.hashHead {
	1220	d.hashHead[i] = 0
	1221	}
	1222	for i := range d.hashPrev {
	1223	d.hashPrev[i] = 0
	1224	}
	1225	d.hashOffset = 1
	1226	d.index, d.windowEnd = 0, 0
	1227	d.blockStart, d.byteAvailable = 0, false
	1228	d.tokens.n = 0
	1229	d.length = minMatchLength - 1
	1230	d.offset = 0
	1231	d.hash = 0
	1232	d.ii = 0
	1233	d.maxInsertIndex = 0
	1234	}
	1235	}
	1236
	1237	func (d *compressor) close() error {
	1238	if d.err != nil {
	1239	return d.err
	1240	}
	1241	d.sync = true
	1242	d.step(d)
	1243	if d.err != nil {
	1244	return d.err
	1245	}
	1246	if d.w.writeStoredHeader(0, true); d.w.err != nil {
	1247	return d.w.err
	1248	}
	1249	d.w.flush()
	1250	return d.w.err
	1251	}
	1252
	1253	// NewWriter returns a new Writer compressing data at the given level.
	1254	// Following zlib, levels range from 1 (BestSpeed) to 9 (BestCompression);
	1255	// higher levels typically run slower but compress more.
	1256	// Level 0 (NoCompression) does not attempt any compression; it only adds the
	1257	// necessary DEFLATE framing.
	1258	// Level -1 (DefaultCompression) uses the default compression level.
	1259	// Level -2 (ConstantCompression) will use Huffman compression only, giving
	1260	// a very fast compression for all types of input, but sacrificing considerable
	1261	// compression efficiency.
	1262	//
	1263	// If level is in the range [-2, 9] then the error returned will be nil.
	1264	// Otherwise the error returned will be non-nil.
	1265	func NewWriter(w io.Writer, level int) (*Writer, error) {
	1266	var dw Writer
	1267	if err := dw.d.init(w, level); err != nil {
	1268	return nil, err
	1269	}
	1270	return &dw, nil
	1271	}
	1272
	1273	// NewWriterDict is like NewWriter but initializes the new
	1274	// Writer with a preset dictionary. The returned Writer behaves
	1275	// as if the dictionary had been written to it without producing
	1276	// any compressed output. The compressed data written to w
	1277	// can only be decompressed by a Reader initialized with the
	1278	// same dictionary.
	1279	func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, error) {
	1280	dw := &dictWriter{w}
	1281	zw, err := NewWriter(dw, level)
	1282	if err != nil {
	1283	return nil, err
	1284	}
	1285	zw.d.fillWindow(dict)
	1286	zw.dict = append(zw.dict, dict...) // duplicate dictionary for Reset method.
	1287	return zw, err
	1288	}
	1289
	1290	type dictWriter struct {
	1291	w io.Writer
	1292	}
	1293
	1294	func (w *dictWriter) Write(b []byte) (n int, err error) {
	1295	return w.w.Write(b)
	1296	}
	1297
	1298	// A Writer takes data written to it and writes the compressed
	1299	// form of that data to an underlying writer (see NewWriter).
	1300	type Writer struct {
	1301	d compressor
	1302	dict []byte
	1303	}
	1304
	1305	// Write writes data to w, which will eventually write the
	1306	// compressed form of data to its underlying writer.
	1307	func (w *Writer) Write(data []byte) (n int, err error) {
	1308	return w.d.write(data)
	1309	}
	1310
	1311	// Flush flushes any pending data to the underlying writer.
	1312	// It is useful mainly in compressed network protocols, to ensure that
	1313	// a remote reader has enough data to reconstruct a packet.
	1314	// Flush does not return until the data has been written.
	1315	// Calling Flush when there is no pending data still causes the Writer
	1316	// to emit a sync marker of at least 4 bytes.
	1317	// If the underlying writer returns an error, Flush returns that error.
	1318	//
	1319	// In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH.
	1320	func (w *Writer) Flush() error {
	1321	// For more about flushing:
	1322	// http://www.bolet.org/~pornin/deflate-flush.html
	1323	return w.d.syncFlush()
	1324	}
	1325
	1326	// Close flushes and closes the writer.
	1327	func (w *Writer) Close() error {
	1328	return w.d.close()
	1329	}
	1330
	1331	// Reset discards the writer's state and makes it equivalent to
	1332	// the result of NewWriter or NewWriterDict called with dst
	1333	// and w's level and dictionary.
	1334	func (w *Writer) Reset(dst io.Writer) {
	1335	if dw, ok := w.d.w.writer.(*dictWriter); ok {
	1336	// w was created with NewWriterDict
	1337	dw.w = dst
	1338	w.d.reset(dw)
	1339	w.d.fillWindow(w.dict)
	1340	} else {
	1341	// w was created with NewWriter
	1342	w.d.reset(dst)
	1343	}
	1344	}
	1345
	1346	// ResetDict discards the writer's state and makes it equivalent to
	1347	// the result of NewWriter or NewWriterDict called with dst
	1348	// and w's level, but sets a specific dictionary.
	1349	func (w *Writer) ResetDict(dst io.Writer, dict []byte) {
	1350	w.dict = dict
	1351	w.d.reset(dst)
	1352	w.d.fillWindow(w.dict)
	1353	}

vendor/github.com/klauspost/compress/flate/dict_decoder.go+184−0

@@ -0,0 +1,184 @@
	1	// Copyright 2016 The Go Authors. All rights reserved.
	2	// Use of this source code is governed by a BSD-style
	3	// license that can be found in the LICENSE file.
	4
	5	package flate
	6
	7	// dictDecoder implements the LZ77 sliding dictionary as used in decompression.
	8	// LZ77 decompresses data through sequences of two forms of commands:
	9	//
	10	// * Literal insertions: Runs of one or more symbols are inserted into the data
	11	// stream as is. This is accomplished through the writeByte method for a
	12	// single symbol, or combinations of writeSlice/writeMark for multiple symbols.
	13	// Any valid stream must start with a literal insertion if no preset dictionary
	14	// is used.
	15	//
	16	// * Backward copies: Runs of one or more symbols are copied from previously
	17	// emitted data. Backward copies come as the tuple (dist, length) where dist
	18	// determines how far back in the stream to copy from and length determines how
	19	// many bytes to copy. Note that it is valid for the length to be greater than
	20	// the distance. Since LZ77 uses forward copies, that situation is used to
	21	// perform a form of run-length encoding on repeated runs of symbols.
	22	// The writeCopy and tryWriteCopy are used to implement this command.
	23	//
	24	// For performance reasons, this implementation performs little to no sanity
	25	// checks about the arguments. As such, the invariants documented for each
	26	// method call must be respected.
	27	type dictDecoder struct {
	28	hist []byte // Sliding window history
	29
	30	// Invariant: 0 <= rdPos <= wrPos <= len(hist)
	31	wrPos int // Current output position in buffer
	32	rdPos int // Have emitted hist[:rdPos] already
	33	full bool // Has a full window length been written yet?
	34	}
	35
	36	// init initializes dictDecoder to have a sliding window dictionary of the given
	37	// size. If a preset dict is provided, it will initialize the dictionary with
	38	// the contents of dict.
	39	func (dd *dictDecoder) init(size int, dict []byte) {
	40	*dd = dictDecoder{hist: dd.hist}
	41
	42	if cap(dd.hist) < size {
	43	dd.hist = make([]byte, size)
	44	}
	45	dd.hist = dd.hist[:size]
	46
	47	if len(dict) > len(dd.hist) {
	48	dict = dict[len(dict)-len(dd.hist):]
	49	}
	50	dd.wrPos = copy(dd.hist, dict)
	51	if dd.wrPos == len(dd.hist) {
	52	dd.wrPos = 0
	53	dd.full = true
	54	}
	55	dd.rdPos = dd.wrPos
	56	}
	57
	58	// histSize reports the total amount of historical data in the dictionary.
	59	func (dd *dictDecoder) histSize() int {
	60	if dd.full {
	61	return len(dd.hist)
	62	}
	63	return dd.wrPos
	64	}
	65
	66	// availRead reports the number of bytes that can be flushed by readFlush.
	67	func (dd *dictDecoder) availRead() int {
	68	return dd.wrPos - dd.rdPos
	69	}
	70
	71	// availWrite reports the available amount of output buffer space.
	72	func (dd *dictDecoder) availWrite() int {
	73	return len(dd.hist) - dd.wrPos
	74	}
	75
	76	// writeSlice returns a slice of the available buffer to write data to.
	77	//
	78	// This invariant will be kept: len(s) <= availWrite()
	79	func (dd *dictDecoder) writeSlice() []byte {
	80	return dd.hist[dd.wrPos:]
	81	}
	82
	83	// writeMark advances the writer pointer by cnt.
	84	//
	85	// This invariant must be kept: 0 <= cnt <= availWrite()
	86	func (dd *dictDecoder) writeMark(cnt int) {
	87	dd.wrPos += cnt
	88	}
	89
	90	// writeByte writes a single byte to the dictionary.
	91	//
	92	// This invariant must be kept: 0 < availWrite()
	93	func (dd *dictDecoder) writeByte(c byte) {
	94	dd.hist[dd.wrPos] = c
	95	dd.wrPos++
	96	}
	97
	98	// writeCopy copies a string at a given (dist, length) to the output.
	99	// This returns the number of bytes copied and may be less than the requested
	100	// length if the available space in the output buffer is too small.
	101	//
	102	// This invariant must be kept: 0 < dist <= histSize()
	103	func (dd *dictDecoder) writeCopy(dist, length int) int {
	104	dstBase := dd.wrPos
	105	dstPos := dstBase
	106	srcPos := dstPos - dist
	107	endPos := dstPos + length
	108	if endPos > len(dd.hist) {
	109	endPos = len(dd.hist)
	110	}
	111
	112	// Copy non-overlapping section after destination position.
	113	//
	114	// This section is non-overlapping in that the copy length for this section
	115	// is always less than or equal to the backwards distance. This can occur
	116	// if a distance refers to data that wraps-around in the buffer.
	117	// Thus, a backwards copy is performed here; that is, the exact bytes in
	118	// the source prior to the copy is placed in the destination.
	119	if srcPos < 0 {
	120	srcPos += len(dd.hist)
	121	dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:])
	122	srcPos = 0
	123	}
	124
	125	// Copy possibly overlapping section before destination position.
	126	//
	127	// This section can overlap if the copy length for this section is larger
	128	// than the backwards distance. This is allowed by LZ77 so that repeated
	129	// strings can be succinctly represented using (dist, length) pairs.
	130	// Thus, a forwards copy is performed here; that is, the bytes copied is
	131	// possibly dependent on the resulting bytes in the destination as the copy
	132	// progresses along. This is functionally equivalent to the following:
	133	//
	134	// for i := 0; i < endPos-dstPos; i++ {
	135	// dd.hist[dstPos+i] = dd.hist[srcPos+i]
	136	// }
	137	// dstPos = endPos
	138	//
	139	for dstPos < endPos {
	140	dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:dstPos])
	141	}
	142
	143	dd.wrPos = dstPos
	144	return dstPos - dstBase
	145	}
	146
	147	// tryWriteCopy tries to copy a string at a given (distance, length) to the
	148	// output. This specialized version is optimized for short distances.
	149	//
	150	// This method is designed to be inlined for performance reasons.
	151	//
	152	// This invariant must be kept: 0 < dist <= histSize()
	153	func (dd *dictDecoder) tryWriteCopy(dist, length int) int {
	154	dstPos := dd.wrPos
	155	endPos := dstPos + length
	156	if dstPos < dist \|\| endPos > len(dd.hist) {
	157	return 0
	158	}
	159	dstBase := dstPos
	160	srcPos := dstPos - dist
	161
	162	// Copy possibly overlapping section before destination position.
	163	loop:
	164	dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:dstPos])
	165	if dstPos < endPos {
	166	goto loop // Avoid for-loop so that this function can be inlined
	167	}
	168
	169	dd.wrPos = dstPos
	170	return dstPos - dstBase
	171	}
	172
	173	// readFlush returns a slice of the historical buffer that is ready to be
	174	// emitted to the user. The data returned by readFlush must be fully consumed
	175	// before calling any other dictDecoder methods.
	176	func (dd *dictDecoder) readFlush() []byte {
	177	toRead := dd.hist[dd.rdPos:dd.wrPos]
	178	dd.rdPos = dd.wrPos
	179	if dd.wrPos == len(dd.hist) {
	180	dd.wrPos, dd.rdPos = 0, 0
	181	dd.full = true
	182	}
	183	return toRead
	184	}

vendor/github.com/klauspost/compress/flate/gen.go+265−0

@@ -0,0 +1,265 @@
	1	// Copyright 2012 The Go Authors. All rights reserved.
	2	// Use of this source code is governed by a BSD-style
	3	// license that can be found in the LICENSE file.
	4
	5	// +build ignore
	6
	7	// This program generates fixedhuff.go
	8	// Invoke as
	9	//
	10	// go run gen.go -output fixedhuff.go
	11
	12	package main
	13
	14	import (
	15	"bytes"
	16	"flag"
	17	"fmt"
	18	"go/format"
	19	"io/ioutil"
	20	"log"
	21	)
	22
	23	var filename = flag.String("output", "fixedhuff.go", "output file name")
	24
	25	const maxCodeLen = 16
	26
	27	// Note: the definition of the huffmanDecoder struct is copied from
	28	// inflate.go, as it is private to the implementation.
	29
	30	// chunk & 15 is number of bits
	31	// chunk >> 4 is value, including table link
	32
	33	const (
	34	huffmanChunkBits = 9
	35	huffmanNumChunks = 1 << huffmanChunkBits
	36	huffmanCountMask = 15
	37	huffmanValueShift = 4
	38	)
	39
	40	type huffmanDecoder struct {
	41	min int // the minimum code length
	42	chunks [huffmanNumChunks]uint32 // chunks as described above
	43	links [][]uint32 // overflow links
	44	linkMask uint32 // mask the width of the link table
	45	}
	46
	47	// Initialize Huffman decoding tables from array of code lengths.
	48	// Following this function, h is guaranteed to be initialized into a complete
	49	// tree (i.e., neither over-subscribed nor under-subscribed). The exception is a
	50	// degenerate case where the tree has only a single symbol with length 1. Empty
	51	// trees are permitted.
	52	func (h *huffmanDecoder) init(bits []int) bool {
	53	// Sanity enables additional runtime tests during Huffman
	54	// table construction. It's intended to be used during
	55	// development to supplement the currently ad-hoc unit tests.
	56	const sanity = false
	57
	58	if h.min != 0 {
	59	*h = huffmanDecoder{}
	60	}
	61
	62	// Count number of codes of each length,
	63	// compute min and max length.
	64	var count [maxCodeLen]int
	65	var min, max int
	66	for _, n := range bits {
	67	if n == 0 {
	68	continue
	69	}
	70	if min == 0 \|\| n < min {
	71	min = n
	72	}
	73	if n > max {
	74	max = n
	75	}
	76	count[n]++
	77	}
	78
	79	// Empty tree. The decompressor.huffSym function will fail later if the tree
	80	// is used. Technically, an empty tree is only valid for the HDIST tree and
	81	// not the HCLEN and HLIT tree. However, a stream with an empty HCLEN tree
	82	// is guaranteed to fail since it will attempt to use the tree to decode the
	83	// codes for the HLIT and HDIST trees. Similarly, an empty HLIT tree is
	84	// guaranteed to fail later since the compressed data section must be
	85	// composed of at least one symbol (the end-of-block marker).
	86	if max == 0 {
	87	return true
	88	}
	89
	90	code := 0
	91	var nextcode [maxCodeLen]int
	92	for i := min; i <= max; i++ {
	93	code <<= 1
	94	nextcode[i] = code
	95	code += count[i]
	96	}
	97
	98	// Check that the coding is complete (i.e., that we've
	99	// assigned all 2-to-the-max possible bit sequences).
	100	// Exception: To be compatible with zlib, we also need to
	101	// accept degenerate single-code codings. See also
	102	// TestDegenerateHuffmanCoding.
	103	if code != 1<<uint(max) && !(code == 1 && max == 1) {
	104	return false
	105	}
	106
	107	h.min = min
	108	if max > huffmanChunkBits {
	109	numLinks := 1 << (uint(max) - huffmanChunkBits)
	110	h.linkMask = uint32(numLinks - 1)
	111
	112	// create link tables
	113	link := nextcode[huffmanChunkBits+1] >> 1
	114	h.links = make([][]uint32, huffmanNumChunks-link)
	115	for j := uint(link); j < huffmanNumChunks; j++ {
	116	reverse := int(reverseByte[j>>8]) \| int(reverseByte[j&0xff])<<8
	117	reverse >>= uint(16 - huffmanChunkBits)
	118	off := j - uint(link)
	119	if sanity && h.chunks[reverse] != 0 {
	120	panic("impossible: overwriting existing chunk")
	121	}
	122	h.chunks[reverse] = uint32(off<<huffmanValueShift \| (huffmanChunkBits + 1))
	123	h.links[off] = make([]uint32, numLinks)
	124	}
	125	}
	126
	127	for i, n := range bits {
	128	if n == 0 {
	129	continue
	130	}
	131	code := nextcode[n]
	132	nextcode[n]++
	133	chunk := uint32(i<<huffmanValueShift \| n)
	134	reverse := int(reverseByte[code>>8]) \| int(reverseByte[code&0xff])<<8
	135	reverse >>= uint(16 - n)
	136	if n <= huffmanChunkBits {
	137	for off := reverse; off < len(h.chunks); off += 1 << uint(n) {
	138	// We should never need to overwrite
	139	// an existing chunk. Also, 0 is
	140	// never a valid chunk, because the
	141	// lower 4 "count" bits should be
	142	// between 1 and 15.
	143	if sanity && h.chunks[off] != 0 {
	144	panic("impossible: overwriting existing chunk")
	145	}
	146	h.chunks[off] = chunk
	147	}
	148	} else {
	149	j := reverse & (huffmanNumChunks - 1)
	150	if sanity && h.chunks[j]&huffmanCountMask != huffmanChunkBits+1 {
	151	// Longer codes should have been
	152	// associated with a link table above.
	153	panic("impossible: not an indirect chunk")
	154	}
	155	value := h.chunks[j] >> huffmanValueShift
	156	linktab := h.links[value]
	157	reverse >>= huffmanChunkBits
	158	for off := reverse; off < len(linktab); off += 1 << uint(n-huffmanChunkBits) {
	159	if sanity && linktab[off] != 0 {
	160	panic("impossible: overwriting existing chunk")
	161	}
	162	linktab[off] = chunk
	163	}
	164	}
	165	}
	166
	167	if sanity {
	168	// Above we've sanity checked that we never overwrote
	169	// an existing entry. Here we additionally check that
	170	// we filled the tables completely.
	171	for i, chunk := range h.chunks {
	172	if chunk == 0 {
	173	// As an exception, in the degenerate
	174	// single-code case, we allow odd
	175	// chunks to be missing.
	176	if code == 1 && i%2 == 1 {
	177	continue
	178	}
	179	panic("impossible: missing chunk")
	180	}
	181	}
	182	for _, linktab := range h.links {
	183	for _, chunk := range linktab {
	184	if chunk == 0 {
	185	panic("impossible: missing chunk")
	186	}
	187	}
	188	}
	189	}
	190
	191	return true
	192	}
	193
	194	func main() {
	195	flag.Parse()
	196
	197	var h huffmanDecoder
	198	var bits [288]int
	199	initReverseByte()
	200	for i := 0; i < 144; i++ {
	201	bits[i] = 8
	202	}
	203	for i := 144; i < 256; i++ {
	204	bits[i] = 9
	205	}
	206	for i := 256; i < 280; i++ {
	207	bits[i] = 7
	208	}
	209	for i := 280; i < 288; i++ {
	210	bits[i] = 8
	211	}
	212	h.init(bits[:])
	213	if h.links != nil {
	214	log.Fatal("Unexpected links table in fixed Huffman decoder")
	215	}
	216
	217	var buf bytes.Buffer
	218
	219	fmt.Fprintf(&buf, `// Copyright 2013 The Go Authors. All rights reserved.
	220	// Use of this source code is governed by a BSD-style
	221	// license that can be found in the LICENSE file.`+"\n\n")
	222
	223	fmt.Fprintln(&buf, "package flate")
	224	fmt.Fprintln(&buf)
	225	fmt.Fprintln(&buf, "// autogenerated by go run gen.go -output fixedhuff.go, DO NOT EDIT")
	226	fmt.Fprintln(&buf)
	227	fmt.Fprintln(&buf, "var fixedHuffmanDecoder = huffmanDecoder{")
	228	fmt.Fprintf(&buf, "\t%d,\n", h.min)
	229	fmt.Fprintln(&buf, "\t[huffmanNumChunks]uint32{")
	230	for i := 0; i < huffmanNumChunks; i++ {
	231	if i&7 == 0 {
	232	fmt.Fprintf(&buf, "\t\t")
	233	} else {
	234	fmt.Fprintf(&buf, " ")
	235	}
	236	fmt.Fprintf(&buf, "0x%04x,", h.chunks[i])
	237	if i&7 == 7 {
	238	fmt.Fprintln(&buf)
	239	}
	240	}
	241	fmt.Fprintln(&buf, "\t},")
	242	fmt.Fprintln(&buf, "\tnil, 0,")
	243	fmt.Fprintln(&buf, "}")
	244
	245	data, err := format.Source(buf.Bytes())
	246	if err != nil {
	247	log.Fatal(err)
	248	}
	249	err = ioutil.WriteFile(*filename, data, 0644)
	250	if err != nil {
	251	log.Fatal(err)
	252	}
	253	}
	254
	255	var reverseByte [256]byte
	256
	257	func initReverseByte() {
	258	for x := 0; x < 256; x++ {
	259	var result byte
	260	for i := uint(0); i < 8; i++ {
	261	result \|= byte(((x >> i) & 1) << (7 - i))
	262	}
	263	reverseByte[x] = result
	264	}
	265	}

vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go+701−0

@@ -0,0 +1,701 @@
	1	// Copyright 2009 The Go Authors. All rights reserved.
	2	// Use of this source code is governed by a BSD-style
	3	// license that can be found in the LICENSE file.
	4
	5	package flate
	6
	7	import (
	8	"io"
	9	)
	10
	11	const (
	12	// The largest offset code.
	13	offsetCodeCount = 30
	14
	15	// The special code used to mark the end of a block.
	16	endBlockMarker = 256
	17
	18	// The first length code.
	19	lengthCodesStart = 257
	20
	21	// The number of codegen codes.
	22	codegenCodeCount = 19
	23	badCode = 255
	24
	25	// bufferFlushSize indicates the buffer size
	26	// after which bytes are flushed to the writer.
	27	// Should preferably be a multiple of 6, since
	28	// we accumulate 6 bytes between writes to the buffer.
	29	bufferFlushSize = 240
	30
	31	// bufferSize is the actual output byte buffer size.
	32	// It must have additional headroom for a flush
	33	// which can contain up to 8 bytes.
	34	bufferSize = bufferFlushSize + 8
	35	)
	36
	37	// The number of extra bits needed by length code X - LENGTH_CODES_START.
	38	var lengthExtraBits = []int8{
	39	/* 257 */ 0, 0, 0,
	40	/* 260 */ 0, 0, 0, 0, 0, 1, 1, 1, 1, 2,
	41	/* 270 */ 2, 2, 2, 3, 3, 3, 3, 4, 4, 4,
	42	/* 280 */ 4, 5, 5, 5, 5, 0,
	43	}
	44
	45	// The length indicated by length code X - LENGTH_CODES_START.
	46	var lengthBase = []uint32{
	47	0, 1, 2, 3, 4, 5, 6, 7, 8, 10,
	48	12, 14, 16, 20, 24, 28, 32, 40, 48, 56,
	49	64, 80, 96, 112, 128, 160, 192, 224, 255,
	50	}
	51
	52	// offset code word extra bits.
	53	var offsetExtraBits = []int8{
	54	0, 0, 0, 0, 1, 1, 2, 2, 3, 3,
	55	4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
	56	9, 9, 10, 10, 11, 11, 12, 12, 13, 13,
	57	/* extended window */
	58	14, 14, 15, 15, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20,
	59	}
	60
	61	var offsetBase = []uint32{
	62	/* normal deflate */
	63	0x000000, 0x000001, 0x000002, 0x000003, 0x000004,
	64	0x000006, 0x000008, 0x00000c, 0x000010, 0x000018,
	65	0x000020, 0x000030, 0x000040, 0x000060, 0x000080,
	66	0x0000c0, 0x000100, 0x000180, 0x000200, 0x000300,
	67	0x000400, 0x000600, 0x000800, 0x000c00, 0x001000,
	68	0x001800, 0x002000, 0x003000, 0x004000, 0x006000,
	69
	70	/* extended window */
	71	0x008000, 0x00c000, 0x010000, 0x018000, 0x020000,
	72	0x030000, 0x040000, 0x060000, 0x080000, 0x0c0000,
	73	0x100000, 0x180000, 0x200000, 0x300000,
	74	}
	75
	76	// The odd order in which the codegen code sizes are written.
	77	var codegenOrder = []uint32{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}
	78
	79	type huffmanBitWriter struct {
	80	// writer is the underlying writer.
	81	// Do not use it directly; use the write method, which ensures
	82	// that Write errors are sticky.
	83	writer io.Writer
	84
	85	// Data waiting to be written is bytes[0:nbytes]
	86	// and then the low nbits of bits.
	87	bits uint64
	88	nbits uint
	89	bytes [bufferSize]byte
	90	codegenFreq [codegenCodeCount]int32
	91	nbytes int
	92	literalFreq []int32
	93	offsetFreq []int32
	94	codegen []uint8
	95	literalEncoding *huffmanEncoder
	96	offsetEncoding *huffmanEncoder
	97	codegenEncoding *huffmanEncoder
	98	err error
	99	}
	100
	101	func newHuffmanBitWriter(w io.Writer) *huffmanBitWriter {
	102	return &huffmanBitWriter{
	103	writer: w,
	104	literalFreq: make([]int32, maxNumLit),
	105	offsetFreq: make([]int32, offsetCodeCount),
	106	codegen: make([]uint8, maxNumLit+offsetCodeCount+1),
	107	literalEncoding: newHuffmanEncoder(maxNumLit),
	108	codegenEncoding: newHuffmanEncoder(codegenCodeCount),
	109	offsetEncoding: newHuffmanEncoder(offsetCodeCount),
	110	}
	111	}
	112
	113	func (w *huffmanBitWriter) reset(writer io.Writer) {
	114	w.writer = writer
	115	w.bits, w.nbits, w.nbytes, w.err = 0, 0, 0, nil
	116	w.bytes = [bufferSize]byte{}
	117	}
	118
	119	func (w *huffmanBitWriter) flush() {
	120	if w.err != nil {
	121	w.nbits = 0
	122	return
	123	}
	124	n := w.nbytes
	125	for w.nbits != 0 {
	126	w.bytes[n] = byte(w.bits)
	127	w.bits >>= 8
	128	if w.nbits > 8 { // Avoid underflow
	129	w.nbits -= 8
	130	} else {
	131	w.nbits = 0
	132	}
	133	n++
	134	}
	135	w.bits = 0
	136	w.write(w.bytes[:n])
	137	w.nbytes = 0
	138	}
	139
	140	func (w *huffmanBitWriter) write(b []byte) {
	141	if w.err != nil {
	142	return
	143	}
	144	_, w.err = w.writer.Write(b)
	145	}
	146
	147	func (w *huffmanBitWriter) writeBits(b int32, nb uint) {
	148	if w.err != nil {
	149	return
	150	}
	151	w.bits \|= uint64(b) << w.nbits
	152	w.nbits += nb
	153	if w.nbits >= 48 {
	154	bits := w.bits
	155	w.bits >>= 48
	156	w.nbits -= 48
	157	n := w.nbytes
	158	bytes := w.bytes[n : n+6]
	159	bytes[0] = byte(bits)
	160	bytes[1] = byte(bits >> 8)
	161	bytes[2] = byte(bits >> 16)
	162	bytes[3] = byte(bits >> 24)
	163	bytes[4] = byte(bits >> 32)
	164	bytes[5] = byte(bits >> 40)
	165	n += 6
	166	if n >= bufferFlushSize {
	167	w.write(w.bytes[:n])
	168	n = 0
	169	}
	170	w.nbytes = n
	171	}
	172	}
	173
	174	func (w *huffmanBitWriter) writeBytes(bytes []byte) {
	175	if w.err != nil {
	176	return
	177	}
	178	n := w.nbytes
	179	if w.nbits&7 != 0 {
	180	w.err = InternalError("writeBytes with unfinished bits")
	181	return
	182	}
	183	for w.nbits != 0 {
	184	w.bytes[n] = byte(w.bits)
	185	w.bits >>= 8
	186	w.nbits -= 8
	187	n++
	188	}
	189	if n != 0 {
	190	w.write(w.bytes[:n])
	191	}
	192	w.nbytes = 0
	193	w.write(bytes)
	194	}
	195
	196	// RFC 1951 3.2.7 specifies a special run-length encoding for specifying
	197	// the literal and offset lengths arrays (which are concatenated into a single
	198	// array). This method generates that run-length encoding.
	199	//
	200	// The result is written into the codegen array, and the frequencies
	201	// of each code is written into the codegenFreq array.
	202	// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional
	203	// information. Code badCode is an end marker
	204	//
	205	// numLiterals The number of literals in literalEncoding
	206	// numOffsets The number of offsets in offsetEncoding
	207	// litenc, offenc The literal and offset encoder to use
	208	func (w huffmanBitWriter) generateCodegen(numLiterals int, numOffsets int, litEnc, offEnc huffmanEncoder) {
	209	for i := range w.codegenFreq {
	210	w.codegenFreq[i] = 0
	211	}
	212	// Note that we are using codegen both as a temporary variable for holding
	213	// a copy of the frequencies, and as the place where we put the result.
	214	// This is fine because the output is always shorter than the input used
	215	// so far.
	216	codegen := w.codegen // cache
	217	// Copy the concatenated code sizes to codegen. Put a marker at the end.
	218	cgnl := codegen[:numLiterals]
	219	for i := range cgnl {
	220	cgnl[i] = uint8(litEnc.codes[i].len)
	221	}
	222
	223	cgnl = codegen[numLiterals : numLiterals+numOffsets]
	224	for i := range cgnl {
	225	cgnl[i] = uint8(offEnc.codes[i].len)
	226	}
	227	codegen[numLiterals+numOffsets] = badCode
	228
	229	size := codegen[0]
	230	count := 1
	231	outIndex := 0
	232	for inIndex := 1; size != badCode; inIndex++ {
	233	// INVARIANT: We have seen "count" copies of size that have not yet
	234	// had output generated for them.
	235	nextSize := codegen[inIndex]
	236	if nextSize == size {
	237	count++
	238	continue
	239	}
	240	// We need to generate codegen indicating "count" of size.
	241	if size != 0 {
	242	codegen[outIndex] = size
	243	outIndex++
	244	w.codegenFreq[size]++
	245	count--
	246	for count >= 3 {
	247	n := 6
	248	if n > count {
	249	n = count
	250	}
	251	codegen[outIndex] = 16
	252	outIndex++
	253	codegen[outIndex] = uint8(n - 3)
	254	outIndex++
	255	w.codegenFreq[16]++
	256	count -= n
	257	}
	258	} else {
	259	for count >= 11 {
	260	n := 138
	261	if n > count {
	262	n = count
	263	}
	264	codegen[outIndex] = 18
	265	outIndex++
	266	codegen[outIndex] = uint8(n - 11)
	267	outIndex++
	268	w.codegenFreq[18]++
	269	count -= n
	270	}
	271	if count >= 3 {
	272	// count >= 3 && count <= 10
	273	codegen[outIndex] = 17
	274	outIndex++
	275	codegen[outIndex] = uint8(count - 3)
	276	outIndex++
	277	w.codegenFreq[17]++
	278	count = 0
	279	}
	280	}
	281	count--
	282	for ; count >= 0; count-- {
	283	codegen[outIndex] = size
	284	outIndex++
	285	w.codegenFreq[size]++
	286	}
	287	// Set up invariant for next time through the loop.
	288	size = nextSize
	289	count = 1
	290	}
	291	// Marker indicating the end of the codegen.
	292	codegen[outIndex] = badCode
	293	}
	294
	295	// dynamicSize returns the size of dynamically encoded data in bits.
	296	func (w huffmanBitWriter) dynamicSize(litEnc, offEnc huffmanEncoder, extraBits int) (size, numCodegens int) {
	297	numCodegens = len(w.codegenFreq)
	298	for numCodegens > 4 && w.codegenFreq[codegenOrder[numCodegens-1]] == 0 {
	299	numCodegens--
	300	}
	301	header := 3 + 5 + 5 + 4 + (3 * numCodegens) +
	302	w.codegenEncoding.bitLength(w.codegenFreq[:]) +
	303	int(w.codegenFreq[16])*2 +
	304	int(w.codegenFreq[17])*3 +
	305	int(w.codegenFreq[18])*7
	306	size = header +
	307	litEnc.bitLength(w.literalFreq) +
	308	offEnc.bitLength(w.offsetFreq) +
	309	extraBits
	310
	311	return size, numCodegens
	312	}
	313
	314	// fixedSize returns the size of dynamically encoded data in bits.
	315	func (w *huffmanBitWriter) fixedSize(extraBits int) int {
	316	return 3 +
	317	fixedLiteralEncoding.bitLength(w.literalFreq) +
	318	fixedOffsetEncoding.bitLength(w.offsetFreq) +
	319	extraBits
	320	}
	321
	322	// storedSize calculates the stored size, including header.
	323	// The function returns the size in bits and whether the block
	324	// fits inside a single block.
	325	func (w *huffmanBitWriter) storedSize(in []byte) (int, bool) {
	326	if in == nil {
	327	return 0, false
	328	}
	329	if len(in) <= maxStoreBlockSize {
	330	return (len(in) + 5) * 8, true
	331	}
	332	return 0, false
	333	}
	334
	335	func (w *huffmanBitWriter) writeCode(c hcode) {
	336	if w.err != nil {
	337	return
	338	}
	339	w.bits \|= uint64(c.code) << w.nbits
	340	w.nbits += uint(c.len)
	341	if w.nbits >= 48 {
	342	bits := w.bits
	343	w.bits >>= 48
	344	w.nbits -= 48
	345	n := w.nbytes
	346	bytes := w.bytes[n : n+6]
	347	bytes[0] = byte(bits)
	348	bytes[1] = byte(bits >> 8)
	349	bytes[2] = byte(bits >> 16)
	350	bytes[3] = byte(bits >> 24)
	351	bytes[4] = byte(bits >> 32)
	352	bytes[5] = byte(bits >> 40)
	353	n += 6
	354	if n >= bufferFlushSize {
	355	w.write(w.bytes[:n])
	356	n = 0
	357	}
	358	w.nbytes = n
	359	}
	360	}
	361
	362	// Write the header of a dynamic Huffman block to the output stream.
	363	//
	364	// numLiterals The number of literals specified in codegen
	365	// numOffsets The number of offsets specified in codegen
	366	// numCodegens The number of codegens used in codegen
	367	func (w *huffmanBitWriter) writeDynamicHeader(numLiterals int, numOffsets int, numCodegens int, isEof bool) {
	368	if w.err != nil {
	369	return
	370	}
	371	var firstBits int32 = 4
	372	if isEof {
	373	firstBits = 5
	374	}
	375	w.writeBits(firstBits, 3)
	376	w.writeBits(int32(numLiterals-257), 5)
	377	w.writeBits(int32(numOffsets-1), 5)
	378	w.writeBits(int32(numCodegens-4), 4)
	379
	380	for i := 0; i < numCodegens; i++ {
	381	value := uint(w.codegenEncoding.codes[codegenOrder[i]].len)
	382	w.writeBits(int32(value), 3)
	383	}
	384
	385	i := 0
	386	for {
	387	var codeWord int = int(w.codegen[i])
	388	i++
	389	if codeWord == badCode {
	390	break
	391	}
	392	w.writeCode(w.codegenEncoding.codes[uint32(codeWord)])
	393
	394	switch codeWord {
	395	case 16:
	396	w.writeBits(int32(w.codegen[i]), 2)
	397	i++
	398	break
	399	case 17:
	400	w.writeBits(int32(w.codegen[i]), 3)
	401	i++
	402	break
	403	case 18:
	404	w.writeBits(int32(w.codegen[i]), 7)
	405	i++
	406	break
	407	}
	408	}
	409	}
	410
	411	func (w *huffmanBitWriter) writeStoredHeader(length int, isEof bool) {
	412	if w.err != nil {
	413	return
	414	}
	415	var flag int32
	416	if isEof {
	417	flag = 1
	418	}
	419	w.writeBits(flag, 3)
	420	w.flush()
	421	w.writeBits(int32(length), 16)
	422	w.writeBits(int32(^uint16(length)), 16)
	423	}
	424
	425	func (w *huffmanBitWriter) writeFixedHeader(isEof bool) {
	426	if w.err != nil {
	427	return
	428	}
	429	// Indicate that we are a fixed Huffman block
	430	var value int32 = 2
	431	if isEof {
	432	value = 3
	433	}
	434	w.writeBits(value, 3)
	435	}
	436
	437	// writeBlock will write a block of tokens with the smallest encoding.
	438	// The original input can be supplied, and if the huffman encoded data
	439	// is larger than the original bytes, the data will be written as a
	440	// stored block.
	441	// If the input is nil, the tokens will always be Huffman encoded.
	442	func (w *huffmanBitWriter) writeBlock(tokens []token, eof bool, input []byte) {
	443	if w.err != nil {
	444	return
	445	}
	446
	447	tokens = append(tokens, endBlockMarker)
	448	numLiterals, numOffsets := w.indexTokens(tokens)
	449
	450	var extraBits int
	451	storedSize, storable := w.storedSize(input)
	452	if storable {
	453	// We only bother calculating the costs of the extra bits required by
	454	// the length of offset fields (which will be the same for both fixed
	455	// and dynamic encoding), if we need to compare those two encodings
	456	// against stored encoding.
	457	for lengthCode := lengthCodesStart + 8; lengthCode < numLiterals; lengthCode++ {
	458	// First eight length codes have extra size = 0.
	459	extraBits += int(w.literalFreq[lengthCode]) * int(lengthExtraBits[lengthCode-lengthCodesStart])
	460	}
	461	for offsetCode := 4; offsetCode < numOffsets; offsetCode++ {
	462	// First four offset codes have extra size = 0.
	463	extraBits += int(w.offsetFreq[offsetCode]) * int(offsetExtraBits[offsetCode])
	464	}
	465	}
	466
	467	// Figure out smallest code.
	468	// Fixed Huffman baseline.
	469	var literalEncoding = fixedLiteralEncoding
	470	var offsetEncoding = fixedOffsetEncoding
	471	var size = w.fixedSize(extraBits)
	472
	473	// Dynamic Huffman?
	474	var numCodegens int
	475
	476	// Generate codegen and codegenFrequencies, which indicates how to encode
	477	// the literalEncoding and the offsetEncoding.
	478	w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding)
	479	w.codegenEncoding.generate(w.codegenFreq[:], 7)
	480	dynamicSize, numCodegens := w.dynamicSize(w.literalEncoding, w.offsetEncoding, extraBits)
	481
	482	if dynamicSize < size {
	483	size = dynamicSize
	484	literalEncoding = w.literalEncoding
	485	offsetEncoding = w.offsetEncoding
	486	}
	487
	488	// Stored bytes?
	489	if storable && storedSize < size {
	490	w.writeStoredHeader(len(input), eof)
	491	w.writeBytes(input)
	492	return
	493	}
	494
	495	// Huffman.
	496	if literalEncoding == fixedLiteralEncoding {
	497	w.writeFixedHeader(eof)
	498	} else {
	499	w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof)
	500	}
	501
	502	// Write the tokens.
	503	w.writeTokens(tokens, literalEncoding.codes, offsetEncoding.codes)
	504	}
	505
	506	// writeBlockDynamic encodes a block using a dynamic Huffman table.
	507	// This should be used if the symbols used have a disproportionate
	508	// histogram distribution.
	509	// If input is supplied and the compression savings are below 1/16th of the
	510	// input size the block is stored.
	511	func (w *huffmanBitWriter) writeBlockDynamic(tokens []token, eof bool, input []byte) {
	512	if w.err != nil {
	513	return
	514	}
	515
	516	tokens = append(tokens, endBlockMarker)
	517	numLiterals, numOffsets := w.indexTokens(tokens)
	518
	519	// Generate codegen and codegenFrequencies, which indicates how to encode
	520	// the literalEncoding and the offsetEncoding.
	521	w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding)
	522	w.codegenEncoding.generate(w.codegenFreq[:], 7)
	523	size, numCodegens := w.dynamicSize(w.literalEncoding, w.offsetEncoding, 0)
	524
	525	// Store bytes, if we don't get a reasonable improvement.
	526	if ssize, storable := w.storedSize(input); storable && ssize < (size+size>>4) {
	527	w.writeStoredHeader(len(input), eof)
	528	w.writeBytes(input)
	529	return
	530	}
	531
	532	// Write Huffman table.
	533	w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof)
	534
	535	// Write the tokens.
	536	w.writeTokens(tokens, w.literalEncoding.codes, w.offsetEncoding.codes)
	537	}
	538
	539	// indexTokens indexes a slice of tokens, and updates
	540	// literalFreq and offsetFreq, and generates literalEncoding
	541	// and offsetEncoding.
	542	// The number of literal and offset tokens is returned.
	543	func (w *huffmanBitWriter) indexTokens(tokens []token) (numLiterals, numOffsets int) {
	544	for i := range w.literalFreq {
	545	w.literalFreq[i] = 0
	546	}
	547	for i := range w.offsetFreq {
	548	w.offsetFreq[i] = 0
	549	}
	550
	551	for _, t := range tokens {
	552	if t < matchType {
	553	w.literalFreq[t.literal()]++
	554	continue
	555	}
	556	length := t.length()
	557	offset := t.offset()
	558	w.literalFreq[lengthCodesStart+lengthCode(length)]++
	559	w.offsetFreq[offsetCode(offset)]++
	560	}
	561
	562	// get the number of literals
	563	numLiterals = len(w.literalFreq)
	564	for w.literalFreq[numLiterals-1] == 0 {
	565	numLiterals--
	566	}
	567	// get the number of offsets
	568	numOffsets = len(w.offsetFreq)
	569	for numOffsets > 0 && w.offsetFreq[numOffsets-1] == 0 {
	570	numOffsets--
	571	}
	572	if numOffsets == 0 {
	573	// We haven't found a single match. If we want to go with the dynamic encoding,
	574	// we should count at least one offset to be sure that the offset huffman tree could be encoded.
	575	w.offsetFreq[0] = 1
	576	numOffsets = 1
	577	}
	578	w.literalEncoding.generate(w.literalFreq, 15)
	579	w.offsetEncoding.generate(w.offsetFreq, 15)
	580	return
	581	}
	582
	583	// writeTokens writes a slice of tokens to the output.
	584	// codes for literal and offset encoding must be supplied.
	585	func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode) {
	586	if w.err != nil {
	587	return
	588	}
	589	for _, t := range tokens {
	590	if t < matchType {
	591	w.writeCode(leCodes[t.literal()])
	592	continue
	593	}
	594	// Write the length
	595	length := t.length()
	596	lengthCode := lengthCode(length)
	597	w.writeCode(leCodes[lengthCode+lengthCodesStart])
	598	extraLengthBits := uint(lengthExtraBits[lengthCode])
	599	if extraLengthBits > 0 {
	600	extraLength := int32(length - lengthBase[lengthCode])
	601	w.writeBits(extraLength, extraLengthBits)
	602	}
	603	// Write the offset
	604	offset := t.offset()
	605	offsetCode := offsetCode(offset)
	606	w.writeCode(oeCodes[offsetCode])
	607	extraOffsetBits := uint(offsetExtraBits[offsetCode])
	608	if extraOffsetBits > 0 {
	609	extraOffset := int32(offset - offsetBase[offsetCode])
	610	w.writeBits(extraOffset, extraOffsetBits)
	611	}
	612	}
	613	}
	614
	615	// huffOffset is a static offset encoder used for huffman only encoding.
	616	// It can be reused since we will not be encoding offset values.
	617	var huffOffset *huffmanEncoder
	618
	619	func init() {
	620	w := newHuffmanBitWriter(nil)
	621	w.offsetFreq[0] = 1
	622	huffOffset = newHuffmanEncoder(offsetCodeCount)
	623	huffOffset.generate(w.offsetFreq, 15)
	624	}
	625
	626	// writeBlockHuff encodes a block of bytes as either
	627	// Huffman encoded literals or uncompressed bytes if the
	628	// results only gains very little from compression.
	629	func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte) {
	630	if w.err != nil {
	631	return
	632	}
	633
	634	// Clear histogram
	635	for i := range w.literalFreq {
	636	w.literalFreq[i] = 0
	637	}
	638
	639	// Add everything as literals
	640	histogram(input, w.literalFreq)
	641
	642	w.literalFreq[endBlockMarker] = 1
	643
	644	const numLiterals = endBlockMarker + 1
	645	const numOffsets = 1
	646
	647	w.literalEncoding.generate(w.literalFreq, 15)
	648
	649	// Figure out smallest code.
	650	// Always use dynamic Huffman or Store
	651	var numCodegens int
	652
	653	// Generate codegen and codegenFrequencies, which indicates how to encode
	654	// the literalEncoding and the offsetEncoding.
	655	w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, huffOffset)
	656	w.codegenEncoding.generate(w.codegenFreq[:], 7)
	657	size, numCodegens := w.dynamicSize(w.literalEncoding, huffOffset, 0)
	658
	659	// Store bytes, if we don't get a reasonable improvement.
	660	if ssize, storable := w.storedSize(input); storable && ssize < (size+size>>4) {
	661	w.writeStoredHeader(len(input), eof)
	662	w.writeBytes(input)
	663	return
	664	}
	665
	666	// Huffman.
	667	w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof)
	668	encoding := w.literalEncoding.codes[:257]
	669	n := w.nbytes
	670	for _, t := range input {
	671	// Bitwriting inlined, ~30% speedup
	672	c := encoding[t]
	673	w.bits \|= uint64(c.code) << w.nbits
	674	w.nbits += uint(c.len)
	675	if w.nbits < 48 {
	676	continue
	677	}
	678	// Store 6 bytes
	679	bits := w.bits
	680	w.bits >>= 48
	681	w.nbits -= 48
	682	bytes := w.bytes[n : n+6]
	683	bytes[0] = byte(bits)
	684	bytes[1] = byte(bits >> 8)
	685	bytes[2] = byte(bits >> 16)
	686	bytes[3] = byte(bits >> 24)
	687	bytes[4] = byte(bits >> 32)
	688	bytes[5] = byte(bits >> 40)
	689	n += 6
	690	if n < bufferFlushSize {
	691	continue
	692	}
	693	w.write(w.bytes[:n])
	694	if w.err != nil {
	695	return // Return early in the event of write failures
	696	}
	697	n = 0
	698	}
	699	w.nbytes = n
	700	w.writeCode(encoding[endBlockMarker])
	701	}

vendor/github.com/klauspost/compress/flate/huffman_code.go+344−0

@@ -0,0 +1,344 @@
	1	// Copyright 2009 The Go Authors. All rights reserved.
	2	// Use of this source code is governed by a BSD-style
	3	// license that can be found in the LICENSE file.
	4
	5	package flate
	6
	7	import (
	8	"math"
	9	"sort"
	10	)
	11
	12	// hcode is a huffman code with a bit code and bit length.
	13	type hcode struct {
	14	code, len uint16
	15	}
	16
	17	type huffmanEncoder struct {
	18	codes []hcode
	19	freqcache []literalNode
	20	bitCount [17]int32
	21	lns byLiteral // stored to avoid repeated allocation in generate
	22	lfs byFreq // stored to avoid repeated allocation in generate
	23	}
	24
	25	type literalNode struct {
	26	literal uint16
	27	freq int32
	28	}
	29
	30	// A levelInfo describes the state of the constructed tree for a given depth.
	31	type levelInfo struct {
	32	// Our level. for better printing
	33	level int32
	34
	35	// The frequency of the last node at this level
	36	lastFreq int32
	37
	38	// The frequency of the next character to add to this level
	39	nextCharFreq int32
	40
	41	// The frequency of the next pair (from level below) to add to this level.
	42	// Only valid if the "needed" value of the next lower level is 0.
	43	nextPairFreq int32
	44
	45	// The number of chains remaining to generate for this level before moving
	46	// up to the next level
	47	needed int32
	48	}
	49
	50	// set sets the code and length of an hcode.
	51	func (h *hcode) set(code uint16, length uint16) {
	52	h.len = length
	53	h.code = code
	54	}
	55
	56	func maxNode() literalNode { return literalNode{math.MaxUint16, math.MaxInt32} }
	57
	58	func newHuffmanEncoder(size int) *huffmanEncoder {
	59	return &huffmanEncoder{codes: make([]hcode, size)}
	60	}
	61
	62	// Generates a HuffmanCode corresponding to the fixed literal table
	63	func generateFixedLiteralEncoding() *huffmanEncoder {
	64	h := newHuffmanEncoder(maxNumLit)
	65	codes := h.codes
	66	var ch uint16
	67	for ch = 0; ch < maxNumLit; ch++ {
	68	var bits uint16
	69	var size uint16
	70	switch {
	71	case ch < 144:
	72	// size 8, 000110000 .. 10111111
	73	bits = ch + 48
	74	size = 8
	75	break
	76	case ch < 256:
	77	// size 9, 110010000 .. 111111111
	78	bits = ch + 400 - 144
	79	size = 9
	80	break
	81	case ch < 280:
	82	// size 7, 0000000 .. 0010111
	83	bits = ch - 256
	84	size = 7
	85	break
	86	default:
	87	// size 8, 11000000 .. 11000111
	88	bits = ch + 192 - 280
	89	size = 8
	90	}
	91	codes[ch] = hcode{code: reverseBits(bits, byte(size)), len: size}
	92	}
	93	return h
	94	}
	95
	96	func generateFixedOffsetEncoding() *huffmanEncoder {
	97	h := newHuffmanEncoder(30)
	98	codes := h.codes
	99	for ch := range codes {
	100	codes[ch] = hcode{code: reverseBits(uint16(ch), 5), len: 5}
	101	}
	102	return h
	103	}
	104
	105	var fixedLiteralEncoding *huffmanEncoder = generateFixedLiteralEncoding()
	106	var fixedOffsetEncoding *huffmanEncoder = generateFixedOffsetEncoding()
	107
	108	func (h *huffmanEncoder) bitLength(freq []int32) int {
	109	var total int
	110	for i, f := range freq {
	111	if f != 0 {
	112	total += int(f) * int(h.codes[i].len)
	113	}
	114	}
	115	return total
	116	}
	117
	118	const maxBitsLimit = 16
	119
	120	// Return the number of literals assigned to each bit size in the Huffman encoding
	121	//
	122	// This method is only called when list.length >= 3
	123	// The cases of 0, 1, and 2 literals are handled by special case code.
	124	//
	125	// list An array of the literals with non-zero frequencies
	126	// and their associated frequencies. The array is in order of increasing
	127	// frequency, and has as its last element a special element with frequency
	128	// MaxInt32
	129	// maxBits The maximum number of bits that should be used to encode any literal.
	130	// Must be less than 16.
	131	// return An integer array in which array[i] indicates the number of literals
	132	// that should be encoded in i bits.
	133	func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 {
	134	if maxBits >= maxBitsLimit {
	135	panic("flate: maxBits too large")
	136	}
	137	n := int32(len(list))
	138	list = list[0 : n+1]
	139	list[n] = maxNode()
	140
	141	// The tree can't have greater depth than n - 1, no matter what. This
	142	// saves a little bit of work in some small cases
	143	if maxBits > n-1 {
	144	maxBits = n - 1
	145	}
	146
	147	// Create information about each of the levels.
	148	// A bogus "Level 0" whose sole purpose is so that
	149	// level1.prev.needed==0. This makes level1.nextPairFreq
	150	// be a legitimate value that never gets chosen.
	151	var levels [maxBitsLimit]levelInfo
	152	// leafCounts[i] counts the number of literals at the left
	153	// of ancestors of the rightmost node at level i.
	154	// leafCounts[i][j] is the number of literals at the left
	155	// of the level j ancestor.
	156	var leafCounts [maxBitsLimit][maxBitsLimit]int32
	157
	158	for level := int32(1); level <= maxBits; level++ {
	159	// For every level, the first two items are the first two characters.
	160	// We initialize the levels as if we had already figured this out.
	161	levels[level] = levelInfo{
	162	level: level,
	163	lastFreq: list[1].freq,
	164	nextCharFreq: list[2].freq,
	165	nextPairFreq: list[0].freq + list[1].freq,
	166	}
	167	leafCounts[level][level] = 2
	168	if level == 1 {
	169	levels[level].nextPairFreq = math.MaxInt32
	170	}
	171	}
	172
	173	// We need a total of 2*n - 2 items at top level and have already generated 2.
	174	levels[maxBits].needed = 2*n - 4
	175
	176	level := maxBits
	177	for {
	178	l := &levels[level]
	179	if l.nextPairFreq == math.MaxInt32 && l.nextCharFreq == math.MaxInt32 {
	180	// We've run out of both leafs and pairs.
	181	// End all calculations for this level.
	182	// To make sure we never come back to this level or any lower level,
	183	// set nextPairFreq impossibly large.
	184	l.needed = 0
	185	levels[level+1].nextPairFreq = math.MaxInt32
	186	level++
	187	continue
	188	}
	189
	190	prevFreq := l.lastFreq
	191	if l.nextCharFreq < l.nextPairFreq {
	192	// The next item on this row is a leaf node.
	193	n := leafCounts[level][level] + 1
	194	l.lastFreq = l.nextCharFreq
	195	// Lower leafCounts are the same of the previous node.
	196	leafCounts[level][level] = n
	197	l.nextCharFreq = list[n].freq
	198	} else {
	199	// The next item on this row is a pair from the previous row.
	200	// nextPairFreq isn't valid until we generate two
	201	// more values in the level below
	202	l.lastFreq = l.nextPairFreq
	203	// Take leaf counts from the lower level, except counts[level] remains the same.
	204	copy(leafCounts[level][:level], leafCounts[level-1][:level])
	205	levels[l.level-1].needed = 2
	206	}
	207
	208	if l.needed--; l.needed == 0 {
	209	// We've done everything we need to do for this level.
	210	// Continue calculating one level up. Fill in nextPairFreq
	211	// of that level with the sum of the two nodes we've just calculated on
	212	// this level.
	213	if l.level == maxBits {
	214	// All done!
	215	break
	216	}
	217	levels[l.level+1].nextPairFreq = prevFreq + l.lastFreq
	218	level++
	219	} else {
	220	// If we stole from below, move down temporarily to replenish it.
	221	for levels[level-1].needed > 0 {
	222	level--
	223	}
	224	}
	225	}
	226
	227	// Somethings is wrong if at the end, the top level is null or hasn't used
	228	// all of the leaves.
	229	if leafCounts[maxBits][maxBits] != n {
	230	panic("leafCounts[maxBits][maxBits] != n")
	231	}
	232
	233	bitCount := h.bitCount[:maxBits+1]
	234	bits := 1
	235	counts := &leafCounts[maxBits]
	236	for level := maxBits; level > 0; level-- {
	237	// chain.leafCount gives the number of literals requiring at least "bits"
	238	// bits to encode.
	239	bitCount[bits] = counts[level] - counts[level-1]
	240	bits++
	241	}
	242	return bitCount
	243	}
	244
	245	// Look at the leaves and assign them a bit count and an encoding as specified
	246	// in RFC 1951 3.2.2
	247	func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode) {
	248	code := uint16(0)
	249	for n, bits := range bitCount {
	250	code <<= 1
	251	if n == 0 \|\| bits == 0 {
	252	continue
	253	}
	254	// The literals list[len(list)-bits] .. list[len(list)-bits]
	255	// are encoded using "bits" bits, and get the values
	256	// code, code + 1, .... The code values are
	257	// assigned in literal order (not frequency order).
	258	chunk := list[len(list)-int(bits):]
	259
	260	h.lns.sort(chunk)
	261	for _, node := range chunk {
	262	h.codes[node.literal] = hcode{code: reverseBits(code, uint8(n)), len: uint16(n)}
	263	code++
	264	}
	265	list = list[0 : len(list)-int(bits)]
	266	}
	267	}
	268
	269	// Update this Huffman Code object to be the minimum code for the specified frequency count.
	270	//
	271	// freq An array of frequencies, in which frequency[i] gives the frequency of literal i.
	272	// maxBits The maximum number of bits to use for any literal.
	273	func (h *huffmanEncoder) generate(freq []int32, maxBits int32) {
	274	if h.freqcache == nil {
	275	// Allocate a reusable buffer with the longest possible frequency table.
	276	// Possible lengths are codegenCodeCount, offsetCodeCount and maxNumLit.
	277	// The largest of these is maxNumLit, so we allocate for that case.
	278	h.freqcache = make([]literalNode, maxNumLit+1)
	279	}
	280	list := h.freqcache[:len(freq)+1]
	281	// Number of non-zero literals
	282	count := 0
	283	// Set list to be the set of all non-zero literals and their frequencies
	284	for i, f := range freq {
	285	if f != 0 {
	286	list[count] = literalNode{uint16(i), f}
	287	count++
	288	} else {
	289	list[count] = literalNode{}
	290	h.codes[i].len = 0
	291	}
	292	}
	293	list[len(freq)] = literalNode{}
	294
	295	list = list[:count]
	296	if count <= 2 {
	297	// Handle the small cases here, because they are awkward for the general case code. With
	298	// two or fewer literals, everything has bit length 1.
	299	for i, node := range list {
	300	// "list" is in order of increasing literal value.
	301	h.codes[node.literal].set(uint16(i), 1)
	302	}
	303	return
	304	}
	305	h.lfs.sort(list)
	306
	307	// Get the number of literals for each bit count
	308	bitCount := h.bitCounts(list, maxBits)
	309	// And do the assignment
	310	h.assignEncodingAndSize(bitCount, list)
	311	}
	312
	313	type byLiteral []literalNode
	314
	315	func (s *byLiteral) sort(a []literalNode) {
	316	*s = byLiteral(a)
	317	sort.Sort(s)
	318	}
	319
	320	func (s byLiteral) Len() int { return len(s) }
	321
	322	func (s byLiteral) Less(i, j int) bool {
	323	return s[i].literal < s[j].literal
	324	}
	325
	326	func (s byLiteral) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
	327
	328	type byFreq []literalNode
	329
	330	func (s *byFreq) sort(a []literalNode) {
	331	*s = byFreq(a)
	332	sort.Sort(s)
	333	}
	334
	335	func (s byFreq) Len() int { return len(s) }
	336
	337	func (s byFreq) Less(i, j int) bool {
	338	if s[i].freq == s[j].freq {
	339	return s[i].literal < s[j].literal
	340	}
	341	return s[i].freq < s[j].freq
	342	}
	343
	344	func (s byFreq) Swap(i, j int) { s[i], s[j] = s[j], s[i] }

vendor/github.com/klauspost/compress/flate/inflate.go+868−0

@@ -0,0 +1,868 @@
	1	// Copyright 2009 The Go Authors. All rights reserved.
	2	// Use of this source code is governed by a BSD-style
	3	// license that can be found in the LICENSE file.
	4
	5	// Package flate implements the DEFLATE compressed data format, described in
	6	// RFC 1951. The gzip and zlib packages implement access to DEFLATE-based file
	7	// formats.
	8	package flate
	9
	10	import (
	11	"bufio"
	12	"io"
	13	"strconv"
	14	"sync"
	15	)
	16
	17	const (
	18	maxCodeLen = 16 // max length of Huffman code
	19	maxCodeLenMask = 15 // mask for max length of Huffman code
	20	// The next three numbers come from the RFC section 3.2.7, with the
	21	// additional proviso in section 3.2.5 which implies that distance codes
	22	// 30 and 31 should never occur in compressed data.
	23	maxNumLit = 286
	24	maxNumDist = 30
	25	numCodes = 19 // number of codes in Huffman meta-code
	26	)
	27
	28	// Initialize the fixedHuffmanDecoder only once upon first use.
	29	var fixedOnce sync.Once
	30	var fixedHuffmanDecoder huffmanDecoder
	31
	32	// A CorruptInputError reports the presence of corrupt input at a given offset.
	33	type CorruptInputError int64
	34
	35	func (e CorruptInputError) Error() string {
	36	return "flate: corrupt input before offset " + strconv.FormatInt(int64(e), 10)
	37	}
	38
	39	// An InternalError reports an error in the flate code itself.
	40	type InternalError string
	41
	42	func (e InternalError) Error() string { return "flate: internal error: " + string(e) }
	43
	44	// A ReadError reports an error encountered while reading input.
	45	//
	46	// Deprecated: No longer returned.
	47	type ReadError struct {
	48	Offset int64 // byte offset where error occurred
	49	Err error // error returned by underlying Read
	50	}
	51
	52	func (e *ReadError) Error() string {
	53	return "flate: read error at offset " + strconv.FormatInt(e.Offset, 10) + ": " + e.Err.Error()
	54	}
	55
	56	// A WriteError reports an error encountered while writing output.
	57	//
	58	// Deprecated: No longer returned.
	59	type WriteError struct {
	60	Offset int64 // byte offset where error occurred
	61	Err error // error returned by underlying Write
	62	}
	63
	64	func (e *WriteError) Error() string {
	65	return "flate: write error at offset " + strconv.FormatInt(e.Offset, 10) + ": " + e.Err.Error()
	66	}
	67
	68	// Resetter resets a ReadCloser returned by NewReader or NewReaderDict to
	69	// to switch to a new underlying Reader. This permits reusing a ReadCloser
	70	// instead of allocating a new one.
	71	type Resetter interface {
	72	// Reset discards any buffered data and resets the Resetter as if it was
	73	// newly initialized with the given reader.
	74	Reset(r io.Reader, dict []byte) error
	75	}
	76
	77	// The data structure for decoding Huffman tables is based on that of
	78	// zlib. There is a lookup table of a fixed bit width (huffmanChunkBits),
	79	// For codes smaller than the table width, there are multiple entries
	80	// (each combination of trailing bits has the same value). For codes
	81	// larger than the table width, the table contains a link to an overflow
	82	// table. The width of each entry in the link table is the maximum code
	83	// size minus the chunk width.
	84	//
	85	// Note that you can do a lookup in the table even without all bits
	86	// filled. Since the extra bits are zero, and the DEFLATE Huffman codes
	87	// have the property that shorter codes come before longer ones, the
	88	// bit length estimate in the result is a lower bound on the actual
	89	// number of bits.
	90	//
	91	// See the following:
	92	// http://www.gzip.org/algorithm.txt
	93
	94	// chunk & 15 is number of bits
	95	// chunk >> 4 is value, including table link
	96
	97	const (
	98	huffmanChunkBits = 9
	99	huffmanNumChunks = 1 << huffmanChunkBits
	100	huffmanCountMask = 15
	101	huffmanValueShift = 4
	102	)
	103
	104	type huffmanDecoder struct {
	105	min int // the minimum code length
	106	chunks *[huffmanNumChunks]uint32 // chunks as described above
	107	links [][]uint32 // overflow links
	108	linkMask uint32 // mask the width of the link table
	109	}
	110
	111	// Initialize Huffman decoding tables from array of code lengths.
	112	// Following this function, h is guaranteed to be initialized into a complete
	113	// tree (i.e., neither over-subscribed nor under-subscribed). The exception is a
	114	// degenerate case where the tree has only a single symbol with length 1. Empty
	115	// trees are permitted.
	116	func (h *huffmanDecoder) init(bits []int) bool {
	117	// Sanity enables additional runtime tests during Huffman
	118	// table construction. It's intended to be used during
	119	// development to supplement the currently ad-hoc unit tests.
	120	const sanity = false
	121
	122	if h.chunks == nil {
	123	h.chunks = &[huffmanNumChunks]uint32{}
	124	}
	125	if h.min != 0 {
	126	*h = huffmanDecoder{chunks: h.chunks, links: h.links}
	127	}
	128
	129	// Count number of codes of each length,
	130	// compute min and max length.
	131	var count [maxCodeLen]int
	132	var min, max int
	133	for _, n := range bits {
	134	if n == 0 {
	135	continue
	136	}
	137	if min == 0 \|\| n < min {
	138	min = n
	139	}
	140	if n > max {
	141	max = n
	142	}
	143	count[n&maxCodeLenMask]++
	144	}
	145
	146	// Empty tree. The decompressor.huffSym function will fail later if the tree
	147	// is used. Technically, an empty tree is only valid for the HDIST tree and
	148	// not the HCLEN and HLIT tree. However, a stream with an empty HCLEN tree
	149	// is guaranteed to fail since it will attempt to use the tree to decode the
	150	// codes for the HLIT and HDIST trees. Similarly, an empty HLIT tree is
	151	// guaranteed to fail later since the compressed data section must be
	152	// composed of at least one symbol (the end-of-block marker).
	153	if max == 0 {
	154	return true
	155	}
	156
	157	code := 0
	158	var nextcode [maxCodeLen]int
	159	for i := min; i <= max; i++ {
	160	code <<= 1
	161	nextcode[i&maxCodeLenMask] = code
	162	code += count[i&maxCodeLenMask]
	163	}
	164
	165	// Check that the coding is complete (i.e., that we've
	166	// assigned all 2-to-the-max possible bit sequences).
	167	// Exception: To be compatible with zlib, we also need to
	168	// accept degenerate single-code codings. See also
	169	// TestDegenerateHuffmanCoding.
	170	if code != 1<<uint(max) && !(code == 1 && max == 1) {
	171	return false
	172	}
	173
	174	h.min = min
	175	chunks := h.chunks[:]
	176	for i := range chunks {
	177	chunks[i] = 0
	178	}
	179
	180	if max > huffmanChunkBits {
	181	numLinks := 1 << (uint(max) - huffmanChunkBits)
	182	h.linkMask = uint32(numLinks - 1)
	183
	184	// create link tables
	185	link := nextcode[huffmanChunkBits+1] >> 1
	186	if cap(h.links) < huffmanNumChunks-link {
	187	h.links = make([][]uint32, huffmanNumChunks-link)
	188	} else {
	189	h.links = h.links[:huffmanNumChunks-link]
	190	}
	191	for j := uint(link); j < huffmanNumChunks; j++ {
	192	reverse := int(reverseByte[j>>8]) \| int(reverseByte[j&0xff])<<8
	193	reverse >>= uint(16 - huffmanChunkBits)
	194	off := j - uint(link)
	195	if sanity && h.chunks[reverse] != 0 {
	196	panic("impossible: overwriting existing chunk")
	197	}
	198	h.chunks[reverse] = uint32(off<<huffmanValueShift \| (huffmanChunkBits + 1))
	199	if cap(h.links[off]) < numLinks {
	200	h.links[off] = make([]uint32, numLinks)
	201	} else {
	202	links := h.links[off][:0]
	203	h.links[off] = links[:numLinks]
	204	}
	205	}
	206	} else {
	207	h.links = h.links[:0]
	208	}
	209
	210	for i, n := range bits {
	211	if n == 0 {
	212	continue
	213	}
	214	code := nextcode[n]
	215	nextcode[n]++
	216	chunk := uint32(i<<huffmanValueShift \| n)
	217	reverse := int(reverseByte[code>>8]) \| int(reverseByte[code&0xff])<<8
	218	reverse >>= uint(16 - n)
	219	if n <= huffmanChunkBits {
	220	for off := reverse; off < len(h.chunks); off += 1 << uint(n) {
	221	// We should never need to overwrite
	222	// an existing chunk. Also, 0 is
	223	// never a valid chunk, because the
	224	// lower 4 "count" bits should be
	225	// between 1 and 15.
	226	if sanity && h.chunks[off] != 0 {
	227	panic("impossible: overwriting existing chunk")
	228	}
	229	h.chunks[off] = chunk
	230	}
	231	} else {
	232	j := reverse & (huffmanNumChunks - 1)
	233	if sanity && h.chunks[j]&huffmanCountMask != huffmanChunkBits+1 {
	234	// Longer codes should have been
	235	// associated with a link table above.
	236	panic("impossible: not an indirect chunk")
	237	}
	238	value := h.chunks[j] >> huffmanValueShift
	239	linktab := h.links[value]
	240	reverse >>= huffmanChunkBits
	241	for off := reverse; off < len(linktab); off += 1 << uint(n-huffmanChunkBits) {
	242	if sanity && linktab[off] != 0 {
	243	panic("impossible: overwriting existing chunk")
	244	}
	245	linktab[off] = chunk
	246	}
	247	}
	248	}
	249
	250	if sanity {
	251	// Above we've sanity checked that we never overwrote
	252	// an existing entry. Here we additionally check that
	253	// we filled the tables completely.
	254	for i, chunk := range h.chunks {
	255	if chunk == 0 {
	256	// As an exception, in the degenerate
	257	// single-code case, we allow odd
	258	// chunks to be missing.
	259	if code == 1 && i%2 == 1 {
	260	continue
	261	}
	262	panic("impossible: missing chunk")
	263	}
	264	}
	265	for _, linktab := range h.links {
	266	for _, chunk := range linktab {
	267	if chunk == 0 {
	268	panic("impossible: missing chunk")
	269	}
	270	}
	271	}
	272	}
	273
	274	return true
	275	}
	276
	277	// The actual read interface needed by NewReader.
	278	// If the passed in io.Reader does not also have ReadByte,
	279	// the NewReader will introduce its own buffering.
	280	type Reader interface {
	281	io.Reader
	282	io.ByteReader
	283	}
	284
	285	// Decompress state.
	286	type decompressor struct {
	287	// Input source.
	288	r Reader
	289	roffset int64
	290
	291	// Input bits, in top of b.
	292	b uint32
	293	nb uint
	294
	295	// Huffman decoders for literal/length, distance.
	296	h1, h2 huffmanDecoder
	297
	298	// Length arrays used to define Huffman codes.
	299	bits *[maxNumLit + maxNumDist]int
	300	codebits *[numCodes]int
	301
	302	// Output history, buffer.
	303	dict dictDecoder
	304
	305	// Temporary buffer (avoids repeated allocation).
	306	buf [4]byte
	307
	308	// Next step in the decompression,
	309	// and decompression state.
	310	step func(*decompressor)
	311	stepState int
	312	final bool
	313	err error
	314	toRead []byte
	315	hl, hd *huffmanDecoder
	316	copyLen int
	317	copyDist int
	318	}
	319
	320	func (f *decompressor) nextBlock() {
	321	for f.nb < 1+2 {
	322	if f.err = f.moreBits(); f.err != nil {
	323	return
	324	}
	325	}
	326	f.final = f.b&1 == 1
	327	f.b >>= 1
	328	typ := f.b & 3
	329	f.b >>= 2
	330	f.nb -= 1 + 2
	331	switch typ {
	332	case 0:
	333	f.dataBlock()
	334	case 1:
	335	// compressed, fixed Huffman tables
	336	f.hl = &fixedHuffmanDecoder
	337	f.hd = nil
	338	f.huffmanBlock()
	339	case 2:
	340	// compressed, dynamic Huffman tables
	341	if f.err = f.readHuffman(); f.err != nil {
	342	break
	343	}
	344	f.hl = &f.h1
	345	f.hd = &f.h2
	346	f.huffmanBlock()
	347	default:
	348	// 3 is reserved.
	349	f.err = CorruptInputError(f.roffset)
	350	}
	351	}
	352
	353	func (f *decompressor) Read(b []byte) (int, error) {
	354	for {
	355	if len(f.toRead) > 0 {
	356	n := copy(b, f.toRead)
	357	f.toRead = f.toRead[n:]
	358	if len(f.toRead) == 0 {
	359	return n, f.err
	360	}
	361	return n, nil
	362	}
	363	if f.err != nil {
	364	return 0, f.err
	365	}
	366	f.step(f)
	367	if f.err != nil && len(f.toRead) == 0 {
	368	f.toRead = f.dict.readFlush() // Flush what's left in case of error
	369	}
	370	}
	371	}
	372
	373	// Support the io.WriteTo interface for io.Copy and friends.
	374	func (f *decompressor) WriteTo(w io.Writer) (int64, error) {
	375	total := int64(0)
	376	flushed := false
	377	for {
	378	if len(f.toRead) > 0 {
	379	n, err := w.Write(f.toRead)
	380	total += int64(n)
	381	if err != nil {
	382	f.err = err
	383	return total, err
	384	}
	385	if n != len(f.toRead) {
	386	return total, io.ErrShortWrite
	387	}
	388	f.toRead = f.toRead[:0]
	389	}
	390	if f.err != nil && flushed {
	391	if f.err == io.EOF {
	392	return total, nil
	393	}
	394	return total, f.err
	395	}
	396	if f.err == nil {
	397	f.step(f)
	398	}
	399	if len(f.toRead) == 0 && f.err != nil && !flushed {
	400	f.toRead = f.dict.readFlush() // Flush what's left in case of error
	401	flushed = true
	402	}
	403	}
	404	}
	405
	406	func (f *decompressor) Close() error {
	407	if f.err == io.EOF {
	408	return nil
	409	}
	410	return f.err
	411	}
	412
	413	// RFC 1951 section 3.2.7.
	414	// Compression with dynamic Huffman codes
	415
	416	var codeOrder = [...]int{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}
	417
	418	func (f *decompressor) readHuffman() error {
	419	// HLIT[5], HDIST[5], HCLEN[4].
	420	for f.nb < 5+5+4 {
	421	if err := f.moreBits(); err != nil {
	422	return err
	423	}
	424	}
	425	nlit := int(f.b&0x1F) + 257
	426	if nlit > maxNumLit {
	427	return CorruptInputError(f.roffset)
	428	}
	429	f.b >>= 5
	430	ndist := int(f.b&0x1F) + 1
	431	if ndist > maxNumDist {
	432	return CorruptInputError(f.roffset)
	433	}
	434	f.b >>= 5
	435	nclen := int(f.b&0xF) + 4
	436	// numCodes is 19, so nclen is always valid.
	437	f.b >>= 4
	438	f.nb -= 5 + 5 + 4
	439
	440	// (HCLEN+4)*3 bits: code lengths in the magic codeOrder order.
	441	for i := 0; i < nclen; i++ {
	442	for f.nb < 3 {
	443	if err := f.moreBits(); err != nil {
	444	return err
	445	}
	446	}
	447	f.codebits[codeOrder[i]] = int(f.b & 0x7)
	448	f.b >>= 3
	449	f.nb -= 3
	450	}
	451	for i := nclen; i < len(codeOrder); i++ {
	452	f.codebits[codeOrder[i]] = 0
	453	}
	454	if !f.h1.init(f.codebits[0:]) {
	455	return CorruptInputError(f.roffset)
	456	}
	457
	458	// HLIT + 257 code lengths, HDIST + 1 code lengths,
	459	// using the code length Huffman code.
	460	for i, n := 0, nlit+ndist; i < n; {
	461	x, err := f.huffSym(&f.h1)
	462	if err != nil {
	463	return err
	464	}
	465	if x < 16 {
	466	// Actual length.
	467	f.bits[i] = x
	468	i++
	469	continue
	470	}
	471	// Repeat previous length or zero.
	472	var rep int
	473	var nb uint
	474	var b int
	475	switch x {
	476	default:
	477	return InternalError("unexpected length code")
	478	case 16:
	479	rep = 3
	480	nb = 2
	481	if i == 0 {
	482	return CorruptInputError(f.roffset)
	483	}
	484	b = f.bits[i-1]
	485	case 17:
	486	rep = 3
	487	nb = 3
	488	b = 0
	489	case 18:
	490	rep = 11
	491	nb = 7
	492	b = 0
	493	}
	494	for f.nb < nb {
	495	if err := f.moreBits(); err != nil {
	496	return err
	497	}
	498	}
	499	rep += int(f.b & uint32(1<<nb-1))
	500	f.b >>= nb
	501	f.nb -= nb
	502	if i+rep > n {
	503	return CorruptInputError(f.roffset)
	504	}
	505	for j := 0; j < rep; j++ {
	506	f.bits[i] = b
	507	i++
	508	}
	509	}
	510
	511	if !f.h1.init(f.bits[0:nlit]) \|\| !f.h2.init(f.bits[nlit:nlit+ndist]) {
	512	return CorruptInputError(f.roffset)
	513	}
	514
	515	// As an optimization, we can initialize the min bits to read at a time
	516	// for the HLIT tree to the length of the EOB marker since we know that
	517	// every block must terminate with one. This preserves the property that
	518	// we never read any extra bytes after the end of the DEFLATE stream.
	519	if f.h1.min < f.bits[endBlockMarker] {
	520	f.h1.min = f.bits[endBlockMarker]
	521	}
	522
	523	return nil
	524	}
	525
	526	// Decode a single Huffman block from f.
	527	// hl and hd are the Huffman states for the lit/length values
	528	// and the distance values, respectively. If hd == nil, using the
	529	// fixed distance encoding associated with fixed Huffman blocks.
	530	func (f *decompressor) huffmanBlock() {
	531	const (
	532	stateInit = iota // Zero value must be stateInit
	533	stateDict
	534	)
	535
	536	switch f.stepState {
	537	case stateInit:
	538	goto readLiteral
	539	case stateDict:
	540	goto copyHistory
	541	}
	542
	543	readLiteral:
	544	// Read literal and/or (length, distance) according to RFC section 3.2.3.
	545	{
	546	v, err := f.huffSym(f.hl)
	547	if err != nil {
	548	f.err = err
	549	return
	550	}
	551	var n uint // number of bits extra
	552	var length int
	553	switch {
	554	case v < 256:
	555	f.dict.writeByte(byte(v))
	556	if f.dict.availWrite() == 0 {
	557	f.toRead = f.dict.readFlush()
	558	f.step = (*decompressor).huffmanBlock
	559	f.stepState = stateInit
	560	return
	561	}
	562	goto readLiteral
	563	case v == 256:
	564	f.finishBlock()
	565	return
	566	// otherwise, reference to older data
	567	case v < 265:
	568	length = v - (257 - 3)
	569	n = 0
	570	case v < 269:
	571	length = v2 - (2652 - 11)
	572	n = 1
	573	case v < 273:
	574	length = v4 - (2694 - 19)
	575	n = 2
	576	case v < 277:
	577	length = v8 - (2738 - 35)
	578	n = 3
	579	case v < 281:
	580	length = v16 - (27716 - 67)
	581	n = 4
	582	case v < 285:
	583	length = v32 - (28132 - 131)
	584	n = 5
	585	case v < maxNumLit:
	586	length = 258
	587	n = 0
	588	default:
	589	f.err = CorruptInputError(f.roffset)
	590	return
	591	}
	592	if n > 0 {
	593	for f.nb < n {
	594	if err = f.moreBits(); err != nil {
	595	f.err = err
	596	return
	597	}
	598	}
	599	length += int(f.b & uint32(1<<n-1))
	600	f.b >>= n
	601	f.nb -= n
	602	}
	603
	604	var dist int
	605	if f.hd == nil {
	606	for f.nb < 5 {
	607	if err = f.moreBits(); err != nil {
	608	f.err = err
	609	return
	610	}
	611	}
	612	dist = int(reverseByte[(f.b&0x1F)<<3])
	613	f.b >>= 5
	614	f.nb -= 5
	615	} else {
	616	if dist, err = f.huffSym(f.hd); err != nil {
	617	f.err = err
	618	return
	619	}
	620	}
	621
	622	switch {
	623	case dist < 4:
	624	dist++
	625	case dist < maxNumDist:
	626	nb := uint(dist-2) >> 1
	627	// have 1 bit in bottom of dist, need nb more.
	628	extra := (dist & 1) << nb
	629	for f.nb < nb {
	630	if err = f.moreBits(); err != nil {
	631	f.err = err
	632	return
	633	}
	634	}
	635	extra \|= int(f.b & uint32(1<<nb-1))
	636	f.b >>= nb
	637	f.nb -= nb
	638	dist = 1<<(nb+1) + 1 + extra
	639	default:
	640	f.err = CorruptInputError(f.roffset)
	641	return
	642	}
	643
	644	// No check on length; encoding can be prescient.
	645	if dist > f.dict.histSize() {
	646	f.err = CorruptInputError(f.roffset)
	647	return
	648	}
	649
	650	f.copyLen, f.copyDist = length, dist
	651	goto copyHistory
	652	}
	653
	654	copyHistory:
	655	// Perform a backwards copy according to RFC section 3.2.3.
	656	{
	657	cnt := f.dict.tryWriteCopy(f.copyDist, f.copyLen)
	658	if cnt == 0 {
	659	cnt = f.dict.writeCopy(f.copyDist, f.copyLen)
	660	}
	661	f.copyLen -= cnt
	662
	663	if f.dict.availWrite() == 0 \|\| f.copyLen > 0 {
	664	f.toRead = f.dict.readFlush()
	665	f.step = (*decompressor).huffmanBlock // We need to continue this work
	666	f.stepState = stateDict
	667	return
	668	}
	669	goto readLiteral
	670	}
	671	}
	672
	673	// Copy a single uncompressed data block from input to output.
	674	func (f *decompressor) dataBlock() {
	675	// Uncompressed.
	676	// Discard current half-byte.
	677	f.nb = 0
	678	f.b = 0
	679
	680	// Length then ones-complement of length.
	681	nr, err := io.ReadFull(f.r, f.buf[0:4])
	682	f.roffset += int64(nr)
	683	if err != nil {
	684	if err == io.EOF {
	685	err = io.ErrUnexpectedEOF
	686	}
	687	f.err = err
	688	return
	689	}
	690	n := int(f.buf[0]) \| int(f.buf[1])<<8
	691	nn := int(f.buf[2]) \| int(f.buf[3])<<8
	692	if uint16(nn) != uint16(^n) {
	693	f.err = CorruptInputError(f.roffset)
	694	return
	695	}
	696
	697	if n == 0 {
	698	f.toRead = f.dict.readFlush()
	699	f.finishBlock()
	700	return
	701	}
	702
	703	f.copyLen = n
	704	f.copyData()
	705	}
	706
	707	// copyData copies f.copyLen bytes from the underlying reader into f.hist.
	708	// It pauses for reads when f.hist is full.
	709	func (f *decompressor) copyData() {
	710	buf := f.dict.writeSlice()
	711	if len(buf) > f.copyLen {
	712	buf = buf[:f.copyLen]
	713	}
	714
	715	cnt, err := io.ReadFull(f.r, buf)
	716	f.roffset += int64(cnt)
	717	f.copyLen -= cnt
	718	f.dict.writeMark(cnt)
	719	if err != nil {
	720	if err == io.EOF {
	721	err = io.ErrUnexpectedEOF
	722	}
	723	f.err = err
	724	return
	725	}
	726
	727	if f.dict.availWrite() == 0 \|\| f.copyLen > 0 {
	728	f.toRead = f.dict.readFlush()
	729	f.step = (*decompressor).copyData
	730	return
	731	}
	732	f.finishBlock()
	733	}
	734
	735	func (f *decompressor) finishBlock() {
	736	if f.final {
	737	if f.dict.availRead() > 0 {
	738	f.toRead = f.dict.readFlush()
	739	}
	740	f.err = io.EOF
	741	}
	742	f.step = (*decompressor).nextBlock
	743	}
	744
	745	func (f *decompressor) moreBits() error {
	746	c, err := f.r.ReadByte()
	747	if err != nil {
	748	if err == io.EOF {
	749	err = io.ErrUnexpectedEOF
	750	}
	751	return err
	752	}
	753	f.roffset++
	754	f.b \|= uint32(c) << f.nb
	755	f.nb += 8
	756	return nil
	757	}
	758
	759	// Read the next Huffman-encoded symbol from f according to h.
	760	func (f decompressor) huffSym(h huffmanDecoder) (int, error) {
	761	// Since a huffmanDecoder can be empty or be composed of a degenerate tree
	762	// with single element, huffSym must error on these two edge cases. In both
	763	// cases, the chunks slice will be 0 for the invalid sequence, leading it
	764	// satisfy the n == 0 check below.
	765	n := uint(h.min)
	766	for {
	767	for f.nb < n {
	768	if err := f.moreBits(); err != nil {
	769	return 0, err
	770	}
	771	}
	772	chunk := h.chunks[f.b&(huffmanNumChunks-1)]
	773	n = uint(chunk & huffmanCountMask)
	774	if n > huffmanChunkBits {
	775	chunk = h.links[chunk>>huffmanValueShift][(f.b>>huffmanChunkBits)&h.linkMask]
	776	n = uint(chunk & huffmanCountMask)
	777	}
	778	if n <= f.nb {
	779	if n == 0 {
	780	f.err = CorruptInputError(f.roffset)
	781	return 0, f.err
	782	}
	783	f.b >>= n
	784	f.nb -= n
	785	return int(chunk >> huffmanValueShift), nil
	786	}
	787	}
	788	}
	789
	790	func makeReader(r io.Reader) Reader {
	791	if rr, ok := r.(Reader); ok {
	792	return rr
	793	}
	794	return bufio.NewReader(r)
	795	}
	796
	797	func fixedHuffmanDecoderInit() {
	798	fixedOnce.Do(func() {
	799	// These come from the RFC section 3.2.6.
	800	var bits [288]int
	801	for i := 0; i < 144; i++ {
	802	bits[i] = 8
	803	}
	804	for i := 144; i < 256; i++ {
	805	bits[i] = 9
	806	}
	807	for i := 256; i < 280; i++ {
	808	bits[i] = 7
	809	}
	810	for i := 280; i < 288; i++ {
	811	bits[i] = 8
	812	}
	813	fixedHuffmanDecoder.init(bits[:])
	814	})
	815	}
	816
	817	func (f *decompressor) Reset(r io.Reader, dict []byte) error {
	818	*f = decompressor{
	819	r: makeReader(r),
	820	bits: f.bits,
	821	codebits: f.codebits,
	822	h1: f.h1,
	823	h2: f.h2,
	824	dict: f.dict,
	825	step: (*decompressor).nextBlock,
	826	}
	827	f.dict.init(maxMatchOffset, dict)
	828	return nil
	829	}
	830
	831	// NewReader returns a new ReadCloser that can be used
	832	// to read the uncompressed version of r.
	833	// If r does not also implement io.ByteReader,
	834	// the decompressor may read more data than necessary from r.
	835	// It is the caller's responsibility to call Close on the ReadCloser
	836	// when finished reading.
	837	//
	838	// The ReadCloser returned by NewReader also implements Resetter.
	839	func NewReader(r io.Reader) io.ReadCloser {
	840	fixedHuffmanDecoderInit()
	841
	842	var f decompressor
	843	f.r = makeReader(r)
	844	f.bits = new([maxNumLit + maxNumDist]int)
	845	f.codebits = new([numCodes]int)
	846	f.step = (*decompressor).nextBlock
	847	f.dict.init(maxMatchOffset, nil)
	848	return &f
	849	}
	850
	851	// NewReaderDict is like NewReader but initializes the reader
	852	// with a preset dictionary. The returned Reader behaves as if
	853	// the uncompressed data stream started with the given dictionary,
	854	// which has already been read. NewReaderDict is typically used
	855	// to read data compressed by NewWriterDict.
	856	//
	857	// The ReadCloser returned by NewReader also implements Resetter.
	858	func NewReaderDict(r io.Reader, dict []byte) io.ReadCloser {
	859	fixedHuffmanDecoderInit()
	860
	861	var f decompressor
	862	f.r = makeReader(r)
	863	f.bits = new([maxNumLit + maxNumDist]int)
	864	f.codebits = new([numCodes]int)
	865	f.step = (*decompressor).nextBlock
	866	f.dict.init(maxMatchOffset, dict)
	867	return &f
	868	}

vendor/github.com/klauspost/compress/flate/reverse_bits.go+48−0

@@ -0,0 +1,48 @@
	1	// Copyright 2009 The Go Authors. All rights reserved.
	2	// Use of this source code is governed by a BSD-style
	3	// license that can be found in the LICENSE file.
	4
	5	package flate
	6
	7	var reverseByte = [256]byte{
	8	0x00, 0x80, 0x40, 0xc0, 0x20, 0xa0, 0x60, 0xe0,
	9	0x10, 0x90, 0x50, 0xd0, 0x30, 0xb0, 0x70, 0xf0,
	10	0x08, 0x88, 0x48, 0xc8, 0x28, 0xa8, 0x68, 0xe8,
	11	0x18, 0x98, 0x58, 0xd8, 0x38, 0xb8, 0x78, 0xf8,
	12	0x04, 0x84, 0x44, 0xc4, 0x24, 0xa4, 0x64, 0xe4,
	13	0x14, 0x94, 0x54, 0xd4, 0x34, 0xb4, 0x74, 0xf4,
	14	0x0c, 0x8c, 0x4c, 0xcc, 0x2c, 0xac, 0x6c, 0xec,
	15	0x1c, 0x9c, 0x5c, 0xdc, 0x3c, 0xbc, 0x7c, 0xfc,
	16	0x02, 0x82, 0x42, 0xc2, 0x22, 0xa2, 0x62, 0xe2,
	17	0x12, 0x92, 0x52, 0xd2, 0x32, 0xb2, 0x72, 0xf2,
	18	0x0a, 0x8a, 0x4a, 0xca, 0x2a, 0xaa, 0x6a, 0xea,
	19	0x1a, 0x9a, 0x5a, 0xda, 0x3a, 0xba, 0x7a, 0xfa,
	20	0x06, 0x86, 0x46, 0xc6, 0x26, 0xa6, 0x66, 0xe6,
	21	0x16, 0x96, 0x56, 0xd6, 0x36, 0xb6, 0x76, 0xf6,
	22	0x0e, 0x8e, 0x4e, 0xce, 0x2e, 0xae, 0x6e, 0xee,
	23	0x1e, 0x9e, 0x5e, 0xde, 0x3e, 0xbe, 0x7e, 0xfe,
	24	0x01, 0x81, 0x41, 0xc1, 0x21, 0xa1, 0x61, 0xe1,
	25	0x11, 0x91, 0x51, 0xd1, 0x31, 0xb1, 0x71, 0xf1,
	26	0x09, 0x89, 0x49, 0xc9, 0x29, 0xa9, 0x69, 0xe9,
	27	0x19, 0x99, 0x59, 0xd9, 0x39, 0xb9, 0x79, 0xf9,
	28	0x05, 0x85, 0x45, 0xc5, 0x25, 0xa5, 0x65, 0xe5,
	29	0x15, 0x95, 0x55, 0xd5, 0x35, 0xb5, 0x75, 0xf5,
	30	0x0d, 0x8d, 0x4d, 0xcd, 0x2d, 0xad, 0x6d, 0xed,
	31	0x1d, 0x9d, 0x5d, 0xdd, 0x3d, 0xbd, 0x7d, 0xfd,
	32	0x03, 0x83, 0x43, 0xc3, 0x23, 0xa3, 0x63, 0xe3,
	33	0x13, 0x93, 0x53, 0xd3, 0x33, 0xb3, 0x73, 0xf3,
	34	0x0b, 0x8b, 0x4b, 0xcb, 0x2b, 0xab, 0x6b, 0xeb,
	35	0x1b, 0x9b, 0x5b, 0xdb, 0x3b, 0xbb, 0x7b, 0xfb,
	36	0x07, 0x87, 0x47, 0xc7, 0x27, 0xa7, 0x67, 0xe7,
	37	0x17, 0x97, 0x57, 0xd7, 0x37, 0xb7, 0x77, 0xf7,
	38	0x0f, 0x8f, 0x4f, 0xcf, 0x2f, 0xaf, 0x6f, 0xef,
	39	0x1f, 0x9f, 0x5f, 0xdf, 0x3f, 0xbf, 0x7f, 0xff,
	40	}
	41
	42	func reverseUint16(v uint16) uint16 {
	43	return uint16(reverseByte[v>>8]) \| uint16(reverseByte[v&0xFF])<<8
	44	}
	45
	46	func reverseBits(number uint16, bitLength byte) uint16 {
	47	return reverseUint16(number << uint8(16-bitLength))
	48	}

vendor/github.com/klauspost/compress/flate/snappy.go+900−0

@@ -0,0 +1,900 @@
	1	// Copyright 2011 The Snappy-Go Authors. All rights reserved.
	2	// Modified for deflate by Klaus Post (c) 2015.
	3	// Use of this source code is governed by a BSD-style
	4	// license that can be found in the LICENSE file.
	5
	6	package flate
	7
	8	// emitLiteral writes a literal chunk and returns the number of bytes written.
	9	func emitLiteral(dst *tokens, lit []byte) {
	10	ol := int(dst.n)
	11	for i, v := range lit {
	12	dst.tokens[(i+ol)&maxStoreBlockSize] = token(v)
	13	}
	14	dst.n += uint16(len(lit))
	15	}
	16
	17	// emitCopy writes a copy chunk and returns the number of bytes written.
	18	func emitCopy(dst *tokens, offset, length int) {
	19	dst.tokens[dst.n] = matchToken(uint32(length-3), uint32(offset-minOffsetSize))
	20	dst.n++
	21	}
	22
	23	type snappyEnc interface {
	24	Encode(dst *tokens, src []byte)
	25	Reset()
	26	}
	27
	28	func newSnappy(level int) snappyEnc {
	29	switch level {
	30	case 1:
	31	return &snappyL1{}
	32	case 2:
	33	return &snappyL2{snappyGen: snappyGen{cur: maxStoreBlockSize, prev: make([]byte, 0, maxStoreBlockSize)}}
	34	case 3:
	35	return &snappyL3{snappyGen: snappyGen{cur: maxStoreBlockSize, prev: make([]byte, 0, maxStoreBlockSize)}}
	36	case 4:
	37	return &snappyL4{snappyL3{snappyGen: snappyGen{cur: maxStoreBlockSize, prev: make([]byte, 0, maxStoreBlockSize)}}}
	38	default:
	39	panic("invalid level specified")
	40	}
	41	}
	42
	43	const (
	44	tableBits = 14 // Bits used in the table
	45	tableSize = 1 << tableBits // Size of the table
	46	tableMask = tableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks.
	47	tableShift = 32 - tableBits // Right-shift to get the tableBits most significant bits of a uint32.
	48	baseMatchOffset = 1 // The smallest match offset
	49	baseMatchLength = 3 // The smallest match length per the RFC section 3.2.5
	50	maxMatchOffset = 1 << 15 // The largest match offset
	51	)
	52
	53	func load32(b []byte, i int) uint32 {
	54	b = b[i : i+4 : len(b)] // Help the compiler eliminate bounds checks on the next line.
	55	return uint32(b[0]) \| uint32(b[1])<<8 \| uint32(b[2])<<16 \| uint32(b[3])<<24
	56	}
	57
	58	func load64(b []byte, i int) uint64 {
	59	b = b[i : i+8 : len(b)] // Help the compiler eliminate bounds checks on the next line.
	60	return uint64(b[0]) \| uint64(b[1])<<8 \| uint64(b[2])<<16 \| uint64(b[3])<<24 \|
	61	uint64(b[4])<<32 \| uint64(b[5])<<40 \| uint64(b[6])<<48 \| uint64(b[7])<<56
	62	}
	63
	64	func hash(u uint32) uint32 {
	65	return (u * 0x1e35a7bd) >> tableShift
	66	}
	67
	68	// snappyL1 encapsulates level 1 compression
	69	type snappyL1 struct{}
	70
	71	func (e *snappyL1) Reset() {}
	72
	73	func (e snappyL1) Encode(dst tokens, src []byte) {
	74	const (
	75	inputMargin = 16 - 1
	76	minNonLiteralBlockSize = 1 + 1 + inputMargin
	77	)
	78
	79	// This check isn't in the Snappy implementation, but there, the caller
	80	// instead of the callee handles this case.
	81	if len(src) < minNonLiteralBlockSize {
	82	// We do not fill the token table.
	83	// This will be picked up by caller.
	84	dst.n = uint16(len(src))
	85	return
	86	}
	87
	88	// Initialize the hash table.
	89	//
	90	// The table element type is uint16, as s < sLimit and sLimit < len(src)
	91	// and len(src) <= maxStoreBlockSize and maxStoreBlockSize == 65535.
	92	var table [tableSize]uint16
	93
	94	// sLimit is when to stop looking for offset/length copies. The inputMargin
	95	// lets us use a fast path for emitLiteral in the main loop, while we are
	96	// looking for copies.
	97	sLimit := len(src) - inputMargin
	98
	99	// nextEmit is where in src the next emitLiteral should start from.
	100	nextEmit := 0
	101
	102	// The encoded form must start with a literal, as there are no previous
	103	// bytes to copy, so we start looking for hash matches at s == 1.
	104	s := 1
	105	nextHash := hash(load32(src, s))
	106
	107	for {
	108	// Copied from the C++ snappy implementation:
	109	//
	110	// Heuristic match skipping: If 32 bytes are scanned with no matches
	111	// found, start looking only at every other byte. If 32 more bytes are
	112	// scanned (or skipped), look at every third byte, etc.. When a match
	113	// is found, immediately go back to looking at every byte. This is a
	114	// small loss (~5% performance, ~0.1% density) for compressible data
	115	// due to more bookkeeping, but for non-compressible data (such as
	116	// JPEG) it's a huge win since the compressor quickly "realizes" the
	117	// data is incompressible and doesn't bother looking for matches
	118	// everywhere.
	119	//
	120	// The "skip" variable keeps track of how many bytes there are since
	121	// the last match; dividing it by 32 (ie. right-shifting by five) gives
	122	// the number of bytes to move ahead for each iteration.
	123	skip := 32
	124
	125	nextS := s
	126	candidate := 0
	127	for {
	128	s = nextS
	129	bytesBetweenHashLookups := skip >> 5
	130	nextS = s + bytesBetweenHashLookups
	131	skip += bytesBetweenHashLookups
	132	if nextS > sLimit {
	133	goto emitRemainder
	134	}
	135	candidate = int(table[nextHash&tableMask])
	136	table[nextHash&tableMask] = uint16(s)
	137	nextHash = hash(load32(src, nextS))
	138	if s-candidate <= maxMatchOffset && load32(src, s) == load32(src, candidate) {
	139	break
	140	}
	141	}
	142
	143	// A 4-byte match has been found. We'll later see if more than 4 bytes
	144	// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
	145	// them as literal bytes.
	146	emitLiteral(dst, src[nextEmit:s])
	147
	148	// Call emitCopy, and then see if another emitCopy could be our next
	149	// move. Repeat until we find no match for the input immediately after
	150	// what was consumed by the last emitCopy call.
	151	//
	152	// If we exit this loop normally then we need to call emitLiteral next,
	153	// though we don't yet know how big the literal will be. We handle that
	154	// by proceeding to the next iteration of the main loop. We also can
	155	// exit this loop via goto if we get close to exhausting the input.
	156	for {
	157	// Invariant: we have a 4-byte match at s, and no need to emit any
	158	// literal bytes prior to s.
	159	base := s
	160
	161	// Extend the 4-byte match as long as possible.
	162	//
	163	// This is an inlined version of Snappy's:
	164	// s = extendMatch(src, candidate+4, s+4)
	165	s += 4
	166	s1 := base + maxMatchLength
	167	if s1 > len(src) {
	168	s1 = len(src)
	169	}
	170	a := src[s:s1]
	171	b := src[candidate+4:]
	172	b = b[:len(a)]
	173	l := len(a)
	174	for i := range a {
	175	if a[i] != b[i] {
	176	l = i
	177	break
	178	}
	179	}
	180	s += l
	181
	182	// matchToken is flate's equivalent of Snappy's emitCopy.
	183	dst.tokens[dst.n] = matchToken(uint32(s-base-baseMatchLength), uint32(base-candidate-baseMatchOffset))
	184	dst.n++
	185	nextEmit = s
	186	if s >= sLimit {
	187	goto emitRemainder
	188	}
	189
	190	// We could immediately start working at s now, but to improve
	191	// compression we first update the hash table at s-1 and at s. If
	192	// another emitCopy is not our next move, also calculate nextHash
	193	// at s+1. At least on GOARCH=amd64, these three hash calculations
	194	// are faster as one load64 call (with some shifts) instead of
	195	// three load32 calls.
	196	x := load64(src, s-1)
	197	prevHash := hash(uint32(x >> 0))
	198	table[prevHash&tableMask] = uint16(s - 1)
	199	currHash := hash(uint32(x >> 8))
	200	candidate = int(table[currHash&tableMask])
	201	table[currHash&tableMask] = uint16(s)
	202	if s-candidate > maxMatchOffset \|\| uint32(x>>8) != load32(src, candidate) {
	203	nextHash = hash(uint32(x >> 16))
	204	s++
	205	break
	206	}
	207	}
	208	}
	209
	210	emitRemainder:
	211	if nextEmit < len(src) {
	212	emitLiteral(dst, src[nextEmit:])
	213	}
	214	}
	215
	216	type tableEntry struct {
	217	val uint32
	218	offset int32
	219	}
	220
	221	func load3232(b []byte, i int32) uint32 {
	222	b = b[i : i+4 : len(b)] // Help the compiler eliminate bounds checks on the next line.
	223	return uint32(b[0]) \| uint32(b[1])<<8 \| uint32(b[2])<<16 \| uint32(b[3])<<24
	224	}
	225
	226	func load6432(b []byte, i int32) uint64 {
	227	b = b[i : i+8 : len(b)] // Help the compiler eliminate bounds checks on the next line.
	228	return uint64(b[0]) \| uint64(b[1])<<8 \| uint64(b[2])<<16 \| uint64(b[3])<<24 \|
	229	uint64(b[4])<<32 \| uint64(b[5])<<40 \| uint64(b[6])<<48 \| uint64(b[7])<<56
	230	}
	231
	232	// snappyGen maintains the table for matches,
	233	// and the previous byte block for level 2.
	234	// This is the generic implementation.
	235	type snappyGen struct {
	236	prev []byte
	237	cur int32
	238	}
	239
	240	// snappyGen maintains the table for matches,
	241	// and the previous byte block for level 2.
	242	// This is the generic implementation.
	243	type snappyL2 struct {
	244	snappyGen
	245	table [tableSize]tableEntry
	246	}
	247
	248	// EncodeL2 uses a similar algorithm to level 1, but is capable
	249	// of matching across blocks giving better compression at a small slowdown.
	250	func (e snappyL2) Encode(dst tokens, src []byte) {
	251	const (
	252	inputMargin = 8 - 1
	253	minNonLiteralBlockSize = 1 + 1 + inputMargin
	254	)
	255
	256	// Protect against e.cur wraparound.
	257	if e.cur > 1<<30 {
	258	for i := range e.table[:] {
	259	e.table[i] = tableEntry{}
	260	}
	261	e.cur = maxStoreBlockSize
	262	}
	263
	264	// This check isn't in the Snappy implementation, but there, the caller
	265	// instead of the callee handles this case.
	266	if len(src) < minNonLiteralBlockSize {
	267	// We do not fill the token table.
	268	// This will be picked up by caller.
	269	dst.n = uint16(len(src))
	270	e.cur += maxStoreBlockSize
	271	e.prev = e.prev[:0]
	272	return
	273	}
	274
	275	// sLimit is when to stop looking for offset/length copies. The inputMargin
	276	// lets us use a fast path for emitLiteral in the main loop, while we are
	277	// looking for copies.
	278	sLimit := int32(len(src) - inputMargin)
	279
	280	// nextEmit is where in src the next emitLiteral should start from.
	281	nextEmit := int32(0)
	282	s := int32(0)
	283	cv := load3232(src, s)
	284	nextHash := hash(cv)
	285
	286	for {
	287	// Copied from the C++ snappy implementation:
	288	//
	289	// Heuristic match skipping: If 32 bytes are scanned with no matches
	290	// found, start looking only at every other byte. If 32 more bytes are
	291	// scanned (or skipped), look at every third byte, etc.. When a match
	292	// is found, immediately go back to looking at every byte. This is a
	293	// small loss (~5% performance, ~0.1% density) for compressible data
	294	// due to more bookkeeping, but for non-compressible data (such as
	295	// JPEG) it's a huge win since the compressor quickly "realizes" the
	296	// data is incompressible and doesn't bother looking for matches
	297	// everywhere.
	298	//
	299	// The "skip" variable keeps track of how many bytes there are since
	300	// the last match; dividing it by 32 (ie. right-shifting by five) gives
	301	// the number of bytes to move ahead for each iteration.
	302	skip := int32(32)
	303
	304	nextS := s
	305	var candidate tableEntry
	306	for {
	307	s = nextS
	308	bytesBetweenHashLookups := skip >> 5
	309	nextS = s + bytesBetweenHashLookups
	310	skip += bytesBetweenHashLookups
	311	if nextS > sLimit {
	312	goto emitRemainder
	313	}
	314	candidate = e.table[nextHash&tableMask]
	315	now := load3232(src, nextS)
	316	e.table[nextHash&tableMask] = tableEntry{offset: s + e.cur, val: cv}
	317	nextHash = hash(now)
	318
	319	offset := s - (candidate.offset - e.cur)
	320	if offset > maxMatchOffset \|\| cv != candidate.val {
	321	// Out of range or not matched.
	322	cv = now
	323	continue
	324	}
	325	break
	326	}
	327
	328	// A 4-byte match has been found. We'll later see if more than 4 bytes
	329	// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
	330	// them as literal bytes.
	331	emitLiteral(dst, src[nextEmit:s])
	332
	333	// Call emitCopy, and then see if another emitCopy could be our next
	334	// move. Repeat until we find no match for the input immediately after
	335	// what was consumed by the last emitCopy call.
	336	//
	337	// If we exit this loop normally then we need to call emitLiteral next,
	338	// though we don't yet know how big the literal will be. We handle that
	339	// by proceeding to the next iteration of the main loop. We also can
	340	// exit this loop via goto if we get close to exhausting the input.
	341	for {
	342	// Invariant: we have a 4-byte match at s, and no need to emit any
	343	// literal bytes prior to s.
	344
	345	// Extend the 4-byte match as long as possible.
	346	//
	347	s += 4
	348	t := candidate.offset - e.cur + 4
	349	l := e.matchlen(s, t, src)
	350
	351	// matchToken is flate's equivalent of Snappy's emitCopy. (length,offset)
	352	dst.tokens[dst.n] = matchToken(uint32(l+4-baseMatchLength), uint32(s-t-baseMatchOffset))
	353	dst.n++
	354	s += l
	355	nextEmit = s
	356	if s >= sLimit {
	357	t += l
	358	// Index first pair after match end.
	359	if int(t+4) < len(src) && t > 0 {
	360	cv := load3232(src, t)
	361	e.table[hash(cv)&tableMask] = tableEntry{offset: t + e.cur, val: cv}
	362	}
	363	goto emitRemainder
	364	}
	365
	366	// We could immediately start working at s now, but to improve
	367	// compression we first update the hash table at s-1 and at s. If
	368	// another emitCopy is not our next move, also calculate nextHash
	369	// at s+1. At least on GOARCH=amd64, these three hash calculations
	370	// are faster as one load64 call (with some shifts) instead of
	371	// three load32 calls.
	372	x := load6432(src, s-1)
	373	prevHash := hash(uint32(x))
	374	e.table[prevHash&tableMask] = tableEntry{offset: e.cur + s - 1, val: uint32(x)}
	375	x >>= 8
	376	currHash := hash(uint32(x))
	377	candidate = e.table[currHash&tableMask]
	378	e.table[currHash&tableMask] = tableEntry{offset: e.cur + s, val: uint32(x)}
	379
	380	offset := s - (candidate.offset - e.cur)
	381	if offset > maxMatchOffset \|\| uint32(x) != candidate.val {
	382	cv = uint32(x >> 8)
	383	nextHash = hash(cv)
	384	s++
	385	break
	386	}
	387	}
	388	}
	389
	390	emitRemainder:
	391	if int(nextEmit) < len(src) {
	392	emitLiteral(dst, src[nextEmit:])
	393	}
	394	e.cur += int32(len(src))
	395	e.prev = e.prev[:len(src)]
	396	copy(e.prev, src)
	397	}
	398
	399	type tableEntryPrev struct {
	400	Cur tableEntry
	401	Prev tableEntry
	402	}
	403
	404	// snappyL3
	405	type snappyL3 struct {
	406	snappyGen
	407	table [tableSize]tableEntryPrev
	408	}
	409
	410	// Encode uses a similar algorithm to level 2, will check up to two candidates.
	411	func (e snappyL3) Encode(dst tokens, src []byte) {
	412	const (
	413	inputMargin = 8 - 1
	414	minNonLiteralBlockSize = 1 + 1 + inputMargin
	415	)
	416
	417	// Protect against e.cur wraparound.
	418	if e.cur > 1<<30 {
	419	for i := range e.table[:] {
	420	e.table[i] = tableEntryPrev{}
	421	}
	422	e.snappyGen = snappyGen{cur: maxStoreBlockSize, prev: e.prev[:0]}
	423	}
	424
	425	// This check isn't in the Snappy implementation, but there, the caller
	426	// instead of the callee handles this case.
	427	if len(src) < minNonLiteralBlockSize {
	428	// We do not fill the token table.
	429	// This will be picked up by caller.
	430	dst.n = uint16(len(src))
	431	e.cur += maxStoreBlockSize
	432	e.prev = e.prev[:0]
	433	return
	434	}
	435
	436	// sLimit is when to stop looking for offset/length copies. The inputMargin
	437	// lets us use a fast path for emitLiteral in the main loop, while we are
	438	// looking for copies.
	439	sLimit := int32(len(src) - inputMargin)
	440
	441	// nextEmit is where in src the next emitLiteral should start from.
	442	nextEmit := int32(0)
	443	s := int32(0)
	444	cv := load3232(src, s)
	445	nextHash := hash(cv)
	446
	447	for {
	448	// Copied from the C++ snappy implementation:
	449	//
	450	// Heuristic match skipping: If 32 bytes are scanned with no matches
	451	// found, start looking only at every other byte. If 32 more bytes are
	452	// scanned (or skipped), look at every third byte, etc.. When a match
	453	// is found, immediately go back to looking at every byte. This is a
	454	// small loss (~5% performance, ~0.1% density) for compressible data
	455	// due to more bookkeeping, but for non-compressible data (such as
	456	// JPEG) it's a huge win since the compressor quickly "realizes" the
	457	// data is incompressible and doesn't bother looking for matches
	458	// everywhere.
	459	//
	460	// The "skip" variable keeps track of how many bytes there are since
	461	// the last match; dividing it by 32 (ie. right-shifting by five) gives
	462	// the number of bytes to move ahead for each iteration.
	463	skip := int32(32)
	464
	465	nextS := s
	466	var candidate tableEntry
	467	for {
	468	s = nextS
	469	bytesBetweenHashLookups := skip >> 5
	470	nextS = s + bytesBetweenHashLookups
	471	skip += bytesBetweenHashLookups
	472	if nextS > sLimit {
	473	goto emitRemainder
	474	}
	475	candidates := e.table[nextHash&tableMask]
	476	now := load3232(src, nextS)
	477	e.table[nextHash&tableMask] = tableEntryPrev{Prev: candidates.Cur, Cur: tableEntry{offset: s + e.cur, val: cv}}
	478	nextHash = hash(now)
	479
	480	// Check both candidates
	481	candidate = candidates.Cur
	482	if cv == candidate.val {
	483	offset := s - (candidate.offset - e.cur)
	484	if offset <= maxMatchOffset {
	485	break
	486	}
	487	} else {
	488	// We only check if value mismatches.
	489	// Offset will always be invalid in other cases.
	490	candidate = candidates.Prev
	491	if cv == candidate.val {
	492	offset := s - (candidate.offset - e.cur)
	493	if offset <= maxMatchOffset {
	494	break
	495	}
	496	}
	497	}
	498	cv = now
	499	}
	500
	501	// A 4-byte match has been found. We'll later see if more than 4 bytes
	502	// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
	503	// them as literal bytes.
	504	emitLiteral(dst, src[nextEmit:s])
	505
	506	// Call emitCopy, and then see if another emitCopy could be our next
	507	// move. Repeat until we find no match for the input immediately after
	508	// what was consumed by the last emitCopy call.
	509	//
	510	// If we exit this loop normally then we need to call emitLiteral next,
	511	// though we don't yet know how big the literal will be. We handle that
	512	// by proceeding to the next iteration of the main loop. We also can
	513	// exit this loop via goto if we get close to exhausting the input.
	514	for {
	515	// Invariant: we have a 4-byte match at s, and no need to emit any
	516	// literal bytes prior to s.
	517
	518	// Extend the 4-byte match as long as possible.
	519	//
	520	s += 4
	521	t := candidate.offset - e.cur + 4
	522	l := e.matchlen(s, t, src)
	523
	524	// matchToken is flate's equivalent of Snappy's emitCopy. (length,offset)
	525	dst.tokens[dst.n] = matchToken(uint32(l+4-baseMatchLength), uint32(s-t-baseMatchOffset))
	526	dst.n++
	527	s += l
	528	nextEmit = s
	529	if s >= sLimit {
	530	t += l
	531	// Index first pair after match end.
	532	if int(t+4) < len(src) && t > 0 {
	533	cv := load3232(src, t)
	534	nextHash = hash(cv)
	535	e.table[nextHash&tableMask] = tableEntryPrev{
	536	Prev: e.table[nextHash&tableMask].Cur,
	537	Cur: tableEntry{offset: e.cur + t, val: cv},
	538	}
	539	}
	540	goto emitRemainder
	541	}
	542
	543	// We could immediately start working at s now, but to improve
	544	// compression we first update the hash table at s-3 to s. If
	545	// another emitCopy is not our next move, also calculate nextHash
	546	// at s+1. At least on GOARCH=amd64, these three hash calculations
	547	// are faster as one load64 call (with some shifts) instead of
	548	// three load32 calls.
	549	x := load6432(src, s-3)
	550	prevHash := hash(uint32(x))
	551	e.table[prevHash&tableMask] = tableEntryPrev{
	552	Prev: e.table[prevHash&tableMask].Cur,
	553	Cur: tableEntry{offset: e.cur + s - 3, val: uint32(x)},
	554	}
	555	x >>= 8
	556	prevHash = hash(uint32(x))
	557
	558	e.table[prevHash&tableMask] = tableEntryPrev{
	559	Prev: e.table[prevHash&tableMask].Cur,
	560	Cur: tableEntry{offset: e.cur + s - 2, val: uint32(x)},
	561	}
	562	x >>= 8
	563	prevHash = hash(uint32(x))
	564
	565	e.table[prevHash&tableMask] = tableEntryPrev{
	566	Prev: e.table[prevHash&tableMask].Cur,
	567	Cur: tableEntry{offset: e.cur + s - 1, val: uint32(x)},
	568	}
	569	x >>= 8
	570	currHash := hash(uint32(x))
	571	candidates := e.table[currHash&tableMask]
	572	cv = uint32(x)
	573	e.table[currHash&tableMask] = tableEntryPrev{
	574	Prev: candidates.Cur,
	575	Cur: tableEntry{offset: s + e.cur, val: cv},
	576	}
	577
	578	// Check both candidates
	579	candidate = candidates.Cur
	580	if cv == candidate.val {
	581	offset := s - (candidate.offset - e.cur)
	582	if offset <= maxMatchOffset {
	583	continue
	584	}
	585	} else {
	586	// We only check if value mismatches.
	587	// Offset will always be invalid in other cases.
	588	candidate = candidates.Prev
	589	if cv == candidate.val {
	590	offset := s - (candidate.offset - e.cur)
	591	if offset <= maxMatchOffset {
	592	continue
	593	}
	594	}
	595	}
	596	cv = uint32(x >> 8)
	597	nextHash = hash(cv)
	598	s++
	599	break
	600	}
	601	}
	602
	603	emitRemainder:
	604	if int(nextEmit) < len(src) {
	605	emitLiteral(dst, src[nextEmit:])
	606	}
	607	e.cur += int32(len(src))
	608	e.prev = e.prev[:len(src)]
	609	copy(e.prev, src)
	610	}
	611
	612	// snappyL4
	613	type snappyL4 struct {
	614	snappyL3
	615	}
	616
	617	// Encode uses a similar algorithm to level 3,
	618	// but will check up to two candidates if first isn't long enough.
	619	func (e snappyL4) Encode(dst tokens, src []byte) {
	620	const (
	621	inputMargin = 8 - 3
	622	minNonLiteralBlockSize = 1 + 1 + inputMargin
	623	matchLenGood = 12
	624	)
	625
	626	// Protect against e.cur wraparound.
	627	if e.cur > 1<<30 {
	628	for i := range e.table[:] {
	629	e.table[i] = tableEntryPrev{}
	630	}
	631	e.snappyGen = snappyGen{cur: maxStoreBlockSize, prev: e.prev[:0]}
	632	}
	633
	634	// This check isn't in the Snappy implementation, but there, the caller
	635	// instead of the callee handles this case.
	636	if len(src) < minNonLiteralBlockSize {
	637	// We do not fill the token table.
	638	// This will be picked up by caller.
	639	dst.n = uint16(len(src))
	640	e.cur += maxStoreBlockSize
	641	e.prev = e.prev[:0]
	642	return
	643	}
	644
	645	// sLimit is when to stop looking for offset/length copies. The inputMargin
	646	// lets us use a fast path for emitLiteral in the main loop, while we are
	647	// looking for copies.
	648	sLimit := int32(len(src) - inputMargin)
	649
	650	// nextEmit is where in src the next emitLiteral should start from.
	651	nextEmit := int32(0)
	652	s := int32(0)
	653	cv := load3232(src, s)
	654	nextHash := hash(cv)
	655
	656	for {
	657	// Copied from the C++ snappy implementation:
	658	//
	659	// Heuristic match skipping: If 32 bytes are scanned with no matches
	660	// found, start looking only at every other byte. If 32 more bytes are
	661	// scanned (or skipped), look at every third byte, etc.. When a match
	662	// is found, immediately go back to looking at every byte. This is a
	663	// small loss (~5% performance, ~0.1% density) for compressible data
	664	// due to more bookkeeping, but for non-compressible data (such as
	665	// JPEG) it's a huge win since the compressor quickly "realizes" the
	666	// data is incompressible and doesn't bother looking for matches
	667	// everywhere.
	668	//
	669	// The "skip" variable keeps track of how many bytes there are since
	670	// the last match; dividing it by 32 (ie. right-shifting by five) gives
	671	// the number of bytes to move ahead for each iteration.
	672	skip := int32(32)
	673
	674	nextS := s
	675	var candidate tableEntry
	676	var candidateAlt tableEntry
	677	for {
	678	s = nextS
	679	bytesBetweenHashLookups := skip >> 5
	680	nextS = s + bytesBetweenHashLookups
	681	skip += bytesBetweenHashLookups
	682	if nextS > sLimit {
	683	goto emitRemainder
	684	}
	685	candidates := e.table[nextHash&tableMask]
	686	now := load3232(src, nextS)
	687	e.table[nextHash&tableMask] = tableEntryPrev{Prev: candidates.Cur, Cur: tableEntry{offset: s + e.cur, val: cv}}
	688	nextHash = hash(now)
	689
	690	// Check both candidates
	691	candidate = candidates.Cur
	692	if cv == candidate.val {
	693	offset := s - (candidate.offset - e.cur)
	694	if offset < maxMatchOffset {
	695	offset = s - (candidates.Prev.offset - e.cur)
	696	if cv == candidates.Prev.val && offset < maxMatchOffset {
	697	candidateAlt = candidates.Prev
	698	}
	699	break
	700	}
	701	} else {
	702	// We only check if value mismatches.
	703	// Offset will always be invalid in other cases.
	704	candidate = candidates.Prev
	705	if cv == candidate.val {
	706	offset := s - (candidate.offset - e.cur)
	707	if offset < maxMatchOffset {
	708	break
	709	}
	710	}
	711	}
	712	cv = now
	713	}
	714
	715	// A 4-byte match has been found. We'll later see if more than 4 bytes
	716	// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
	717	// them as literal bytes.
	718	emitLiteral(dst, src[nextEmit:s])
	719
	720	// Call emitCopy, and then see if another emitCopy could be our next
	721	// move. Repeat until we find no match for the input immediately after
	722	// what was consumed by the last emitCopy call.
	723	//
	724	// If we exit this loop normally then we need to call emitLiteral next,
	725	// though we don't yet know how big the literal will be. We handle that
	726	// by proceeding to the next iteration of the main loop. We also can
	727	// exit this loop via goto if we get close to exhausting the input.
	728	for {
	729	// Invariant: we have a 4-byte match at s, and no need to emit any
	730	// literal bytes prior to s.
	731
	732	// Extend the 4-byte match as long as possible.
	733	//
	734	s += 4
	735	t := candidate.offset - e.cur + 4
	736	l := e.matchlen(s, t, src)
	737	// Try alternative candidate if match length < matchLenGood.
	738	if l < matchLenGood-4 && candidateAlt.offset != 0 {
	739	t2 := candidateAlt.offset - e.cur + 4
	740	l2 := e.matchlen(s, t2, src)
	741	if l2 > l {
	742	l = l2
	743	t = t2
	744	}
	745	}
	746	// matchToken is flate's equivalent of Snappy's emitCopy. (length,offset)
	747	dst.tokens[dst.n] = matchToken(uint32(l+4-baseMatchLength), uint32(s-t-baseMatchOffset))
	748	dst.n++
	749	s += l
	750	nextEmit = s
	751	if s >= sLimit {
	752	t += l
	753	// Index first pair after match end.
	754	if int(t+4) < len(src) && t > 0 {
	755	cv := load3232(src, t)
	756	nextHash = hash(cv)
	757	e.table[nextHash&tableMask] = tableEntryPrev{
	758	Prev: e.table[nextHash&tableMask].Cur,
	759	Cur: tableEntry{offset: e.cur + t, val: cv},
	760	}
	761	}
	762	goto emitRemainder
	763	}
	764
	765	// We could immediately start working at s now, but to improve
	766	// compression we first update the hash table at s-3 to s. If
	767	// another emitCopy is not our next move, also calculate nextHash
	768	// at s+1. At least on GOARCH=amd64, these three hash calculations
	769	// are faster as one load64 call (with some shifts) instead of
	770	// three load32 calls.
	771	x := load6432(src, s-3)
	772	prevHash := hash(uint32(x))
	773	e.table[prevHash&tableMask] = tableEntryPrev{
	774	Prev: e.table[prevHash&tableMask].Cur,
	775	Cur: tableEntry{offset: e.cur + s - 3, val: uint32(x)},
	776	}
	777	x >>= 8
	778	prevHash = hash(uint32(x))
	779
	780	e.table[prevHash&tableMask] = tableEntryPrev{
	781	Prev: e.table[prevHash&tableMask].Cur,
	782	Cur: tableEntry{offset: e.cur + s - 2, val: uint32(x)},
	783	}
	784	x >>= 8
	785	prevHash = hash(uint32(x))
	786
	787	e.table[prevHash&tableMask] = tableEntryPrev{
	788	Prev: e.table[prevHash&tableMask].Cur,
	789	Cur: tableEntry{offset: e.cur + s - 1, val: uint32(x)},
	790	}
	791	x >>= 8
	792	currHash := hash(uint32(x))
	793	candidates := e.table[currHash&tableMask]
	794	cv = uint32(x)
	795	e.table[currHash&tableMask] = tableEntryPrev{
	796	Prev: candidates.Cur,
	797	Cur: tableEntry{offset: s + e.cur, val: cv},
	798	}
	799
	800	// Check both candidates
	801	candidate = candidates.Cur
	802	candidateAlt = tableEntry{}
	803	if cv == candidate.val {
	804	offset := s - (candidate.offset - e.cur)
	805	if offset <= maxMatchOffset {
	806	offset = s - (candidates.Prev.offset - e.cur)
	807	if cv == candidates.Prev.val && offset <= maxMatchOffset {
	808	candidateAlt = candidates.Prev
	809	}
	810	continue
	811	}
	812	} else {
	813	// We only check if value mismatches.
	814	// Offset will always be invalid in other cases.
	815	candidate = candidates.Prev
	816	if cv == candidate.val {
	817	offset := s - (candidate.offset - e.cur)
	818	if offset <= maxMatchOffset {
	819	continue
	820	}
	821	}
	822	}
	823	cv = uint32(x >> 8)
	824	nextHash = hash(cv)
	825	s++
	826	break
	827	}
	828	}
	829
	830	emitRemainder:
	831	if int(nextEmit) < len(src) {
	832	emitLiteral(dst, src[nextEmit:])
	833	}
	834	e.cur += int32(len(src))
	835	e.prev = e.prev[:len(src)]
	836	copy(e.prev, src)
	837	}
	838
	839	func (e *snappyGen) matchlen(s, t int32, src []byte) int32 {
	840	s1 := int(s) + maxMatchLength - 4
	841	if s1 > len(src) {
	842	s1 = len(src)
	843	}
	844
	845	// If we are inside the current block
	846	if t >= 0 {
	847	b := src[t:]
	848	a := src[s:s1]
	849	b = b[:len(a)]
	850	// Extend the match to be as long as possible.
	851	for i := range a {
	852	if a[i] != b[i] {
	853	return int32(i)
	854	}
	855	}
	856	return int32(len(a))
	857	}
	858
	859	// We found a match in the previous block.
	860	tp := int32(len(e.prev)) + t
	861	if tp < 0 {
	862	return 0
	863	}
	864
	865	// Extend the match to be as long as possible.
	866	a := src[s:s1]
	867	b := e.prev[tp:]
	868	if len(b) > len(a) {
	869	b = b[:len(a)]
	870	}
	871	a = a[:len(b)]
	872	for i := range b {
	873	if a[i] != b[i] {
	874	return int32(i)
	875	}
	876	}
	877
	878	// If we reached our limit, we matched everything we are
	879	// allowed to in the previous block and we return.
	880	n := int32(len(b))
	881	if int(s+n) == s1 {
	882	return n
	883	}
	884
	885	// Continue looking for more matches in the current block.
	886	a = src[s+n : s1]
	887	b = src[:len(a)]
	888	for i := range a {
	889	if a[i] != b[i] {
	890	return int32(i) + n
	891	}
	892	}
	893	return int32(len(a)) + n
	894	}
	895
	896	// Reset the encoding table.
	897	func (e *snappyGen) Reset() {
	898	e.prev = e.prev[:0]
	899	e.cur += maxMatchOffset
	900	}

vendor/github.com/klauspost/compress/flate/token.go+115−0

@@ -0,0 +1,115 @@
	1	// Copyright 2009 The Go Authors. All rights reserved.
	2	// Use of this source code is governed by a BSD-style
	3	// license that can be found in the LICENSE file.
	4
	5	package flate
	6
	7	import "fmt"
	8
	9	const (
	10	// 2 bits: type 0 = literal 1=EOF 2=Match 3=Unused
	11	// 8 bits: xlength = length - MIN_MATCH_LENGTH
	12	// 22 bits xoffset = offset - MIN_OFFSET_SIZE, or literal
	13	lengthShift = 22
	14	offsetMask = 1<<lengthShift - 1
	15	typeMask = 3 << 30
	16	literalType = 0 << 30
	17	matchType = 1 << 30
	18	)
	19
	20	// The length code for length X (MIN_MATCH_LENGTH <= X <= MAX_MATCH_LENGTH)
	21	// is lengthCodes[length - MIN_MATCH_LENGTH]
	22	var lengthCodes = [...]uint32{
	23	0, 1, 2, 3, 4, 5, 6, 7, 8, 8,
	24	9, 9, 10, 10, 11, 11, 12, 12, 12, 12,
	25	13, 13, 13, 13, 14, 14, 14, 14, 15, 15,
	26	15, 15, 16, 16, 16, 16, 16, 16, 16, 16,
	27	17, 17, 17, 17, 17, 17, 17, 17, 18, 18,
	28	18, 18, 18, 18, 18, 18, 19, 19, 19, 19,
	29	19, 19, 19, 19, 20, 20, 20, 20, 20, 20,
	30	20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
	31	21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
	32	21, 21, 21, 21, 21, 21, 22, 22, 22, 22,
	33	22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
	34	22, 22, 23, 23, 23, 23, 23, 23, 23, 23,
	35	23, 23, 23, 23, 23, 23, 23, 23, 24, 24,
	36	24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
	37	24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
	38	24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
	39	25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
	40	25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
	41	25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
	42	25, 25, 26, 26, 26, 26, 26, 26, 26, 26,
	43	26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
	44	26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
	45	26, 26, 26, 26, 27, 27, 27, 27, 27, 27,
	46	27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
	47	27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
	48	27, 27, 27, 27, 27, 28,
	49	}
	50
	51	var offsetCodes = [...]uint32{
	52	0, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7,
	53	8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9,
	54	10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
	55	11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
	56	12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
	57	12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
	58	13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
	59	13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
	60	14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
	61	14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
	62	14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
	63	14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
	64	15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
	65	15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
	66	15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
	67	15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
	68	}
	69
	70	type token uint32
	71
	72	type tokens struct {
	73	tokens [maxStoreBlockSize + 1]token
	74	n uint16 // Must be able to contain maxStoreBlockSize
	75	}
	76
	77	// Convert a literal into a literal token.
	78	func literalToken(literal uint32) token { return token(literalType + literal) }
	79
	80	// Convert a < xlength, xoffset > pair into a match token.
	81	func matchToken(xlength uint32, xoffset uint32) token {
	82	return token(matchType + xlength<<lengthShift + xoffset)
	83	}
	84
	85	func matchTokend(xlength uint32, xoffset uint32) token {
	86	if xlength > maxMatchLength \|\| xoffset > maxMatchOffset {
	87	panic(fmt.Sprintf("Invalid match: len: %d, offset: %d\n", xlength, xoffset))
	88	return token(matchType)
	89	}
	90	return token(matchType + xlength<<lengthShift + xoffset)
	91	}
	92
	93	// Returns the type of a token
	94	func (t token) typ() uint32 { return uint32(t) & typeMask }
	95
	96	// Returns the literal of a literal token
	97	func (t token) literal() uint32 { return uint32(t - literalType) }
	98
	99	// Returns the extra offset of a match token
	100	func (t token) offset() uint32 { return uint32(t) & offsetMask }
	101
	102	func (t token) length() uint32 { return uint32((t - matchType) >> lengthShift) }
	103
	104	func lengthCode(len uint32) uint32 { return lengthCodes[len] }
	105
	106	// Returns the offset code corresponding to a specific offset
	107	func offsetCode(off uint32) uint32 {
	108	if off < uint32(len(offsetCodes)) {
	109	return offsetCodes[off]
	110	} else if off>>7 < uint32(len(offsetCodes)) {
	111	return offsetCodes[off>>7] + 14
	112	} else {
	113	return offsetCodes[off>>14] + 28
	114	}
	115	}

vendor/github.com/klauspost/pgzip/GO_LICENSE+27−0

@@ -0,0 +1,27 @@
	1	Copyright (c) 2012 The Go Authors. All rights reserved.
	2
	3	Redistribution and use in source and binary forms, with or without
	4	modification, are permitted provided that the following conditions are
	5	met:
	6
	7	* Redistributions of source code must retain the above copyright
	8	notice, this list of conditions and the following disclaimer.
	9	* Redistributions in binary form must reproduce the above
	10	copyright notice, this list of conditions and the following disclaimer
	11	in the documentation and/or other materials provided with the
	12	distribution.
	13	* Neither the name of Google Inc. nor the names of its
	14	contributors may be used to endorse or promote products derived from
	15	this software without specific prior written permission.
	16
	17	THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	18	"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	19	LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	20	A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	21	OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	22	SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	23	LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	24	DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	25	THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	26	(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	27	OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/klauspost/pgzip/LICENSE+22−0

@@ -0,0 +1,22 @@
	1	The MIT License (MIT)
	2
	3	Copyright (c) 2014 Klaus Post
	4
	5	Permission is hereby granted, free of charge, to any person obtaining a copy
	6	of this software and associated documentation files (the "Software"), to deal
	7	in the Software without restriction, including without limitation the rights
	8	to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
	9	copies of the Software, and to permit persons to whom the Software is
	10	furnished to do so, subject to the following conditions:
	11
	12	The above copyright notice and this permission notice shall be included in all
	13	copies or substantial portions of the Software.
	14
	15	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	16	IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	17	FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
	18	AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	19	LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
	20	OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	21	SOFTWARE.
	22

vendor/github.com/klauspost/pgzip/README.md+136−0

@@ -0,0 +1,136 @@
	1	pgzip
	2	=====
	3
	4	Go parallel gzip compression/decompression. This is a fully gzip compatible drop in replacement for "compress/gzip".
	5
	6	This will split compression into blocks that are compressed in parallel.
	7	This can be useful for compressing big amounts of data. The output is a standard gzip file.
	8
	9	The gzip decompression is modified so it decompresses ahead of the current reader.
	10	This means that reads will be non-blocking if the decompressor can keep ahead of your code reading from it.
	11	CRC calculation also takes place in a separate goroutine.
	12
	13	You should only use this if you are (de)compressing big amounts of data,
	14	say more than 1MB at the time, otherwise you will not see any benefit,
	15	and it will likely be faster to use the internal gzip library
	16	or [this package](https://github.com/klauspost/compress).
	17
	18	It is important to note that this library creates and reads standard gzip files.
	19	You do not have to match the compressor/decompressor to get the described speedups,
	20	and the gzip files are fully compatible with other gzip readers/writers.
	21
	22	A golang variant of this is [bgzf](https://godoc.org/github.com/biogo/hts/bgzf),
	23	which has the same feature, as well as seeking in the resulting file.
	24	The only drawback is a slightly bigger overhead compared to this and pure gzip.
	25	See a comparison below.
	26
	27	[![GoDoc][1]][2] [![Build Status][3]][4]
	28
	29	[1]: https://godoc.org/github.com/klauspost/pgzip?status.svg
	30	[2]: https://godoc.org/github.com/klauspost/pgzip
	31	[3]: https://travis-ci.org/klauspost/pgzip.svg
	32	[4]: https://travis-ci.org/klauspost/pgzip
	33
	34	Installation
	35	====
	36	```go get github.com/klauspost/pgzip/...```
	37
	38	You might need to get/update the dependencies:
	39
	40	```
	41	go get -u github.com/klauspost/compress
	42	go get -u github.com/klauspost/crc32
	43	```
	44
	45	Usage
	46	====
	47	[Godoc Doumentation](https://godoc.org/github.com/klauspost/pgzip)
	48
	49	To use as a replacement for gzip, exchange
	50
	51	```import "compress/gzip"```
	52	with
	53	```import gzip "github.com/klauspost/pgzip"```.
	54
	55	# Changes
	56
	57	* Oct 6, 2016: Fixed an issue if the destination writer returned an error.
	58	* Oct 6, 2016: Better buffer reuse, should now generate less garbage.
	59	* Oct 6, 2016: Output does not change based on write sizes.
	60	* Dec 8, 2015: Decoder now supports the io.WriterTo interface, giving a speedup and less GC pressure.
	61	* Oct 9, 2015: Reduced allocations by ~35 by using sync.Pool. ~15% overall speedup.
	62
	63	Changes in [github.com/klauspost/compress](https://github.com/klauspost/compress#changelog) are also carried over, so see that for more changes.
	64
	65	## Compression
	66	The simplest way to use this is to simply do the same as you would when using [compress/gzip](http://golang.org/pkg/compress/gzip).
	67
	68	To change the block size, use the added (*pgzip.Writer).SetConcurrency(blockSize, blocks int) function. With this you can control the approximate size of your blocks, as well as how many you want to be processing in parallel. Default values for this is SetConcurrency(250000, 16), meaning blocks are split at 250000 bytes and up to 16 blocks can be processing at once before the writer blocks.
	69
	70
	71	Example:
	72	```
	73	var b bytes.Buffer
	74	w := gzip.NewWriter(&b)
	75	w.SetConcurrency(100000, 10)
	76	w.Write([]byte("hello, world\n"))
	77	w.Close()
	78	```
	79
	80	To get any performance gains, you should at least be compressing more than 1 megabyte of data at the time.
	81
	82	You should at least have a block size of 100k and at least a number of blocks that match the number of cores your would like to utilize, but about twice the number of blocks would be the best.
	83
	84	Another side effect of this is, that it is likely to speed up your other code, since writes to the compressor only blocks if the compressor is already compressing the number of blocks you have specified. This also means you don't have worry about buffering input to the compressor.
	85
	86	## Decompression
	87
	88	Decompression works similar to compression. That means that you simply call pgzip the same way as you would call [compress/gzip](http://golang.org/pkg/compress/gzip).
	89
	90	The only difference is that if you want to specify your own readahead, you have to use `pgzip.NewReaderN(r io.Reader, blockSize, blocks int)` to get a reader with your custom blocksizes. The `blockSize` is the size of each block decoded, and `blocks` is the maximum number of blocks that is decoded ahead.
	91
	92	See [Example on playground](http://play.golang.org/p/uHv1B5NbDh)
	93
	94	Performance
	95	====
	96	## Compression
	97
	98	See my blog post in [Benchmarks of Golang Gzip](https://blog.klauspost.com/go-gzipdeflate-benchmarks/).
	99
	100	Compression cost is usually about 0.2% with default settings with a block size of 250k.
	101
	102	Example with GOMAXPROC set to 8 (quad core with 8 hyperthreads)
	103
	104	Content is [Matt Mahoneys 10GB corpus](http://mattmahoney.net/dc/10gb.html). Compression level 6.
	105
	106	Compressor \| MB/sec \| speedup \| size \| size overhead (lower=better)
	107	------------\|----------\|---------\|------\|---------
	108	[gzip](http://golang.org/pkg/compress/gzip) (golang) \| 7.21MB/s \| 1.0x \| 4786608902 \| 0%
	109	[gzip](http://github.com/klauspost/compress/gzip) (klauspost) \| 10.98MB/s \| 1.52x \| 4781331645 \| -0.11%
	110	[pgzip](https://github.com/klauspost/pgzip) (klauspost) \| 50.76MB/s\|7.04x \| 4784121440 \| -0.052%
	111	[bgzf](https://godoc.org/github.com/biogo/hts/bgzf) (biogo) \| 38.65MB/s \| 5.36x \| 4924899484 \| 2.889%
	112	[pargzip](https://godoc.org/github.com/golang/build/pargzip) (builder) \| 32.00MB/s \| 4.44x \| 4791226567 \| 0.096%
	113
	114	pgzip also contains a [linear time compression](https://github.com/klauspost/compress#linear-time-compression) mode, that will allow compression at ~150MB per core per second, independent of the content.
	115
	116	See the [complete sheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing) for different content types and compression settings.
	117
	118	## Decompression
	119
	120	The decompression speedup is there because it allows you to do other work while the decompression is taking place.
	121
	122	In the example above, the numbers are as follows on a 4 CPU machine:
	123
	124	Decompressor \| Time \| Speedup
	125	-------------\|------\|--------
	126	[gzip](http://golang.org/pkg/compress/gzip) (golang) \| 1m28.85s \| 0%
	127	[pgzip](https://github.com/klauspost/pgzip) (golang) \| 43.48s \| 104%
	128
	129	But wait, since gzip decompression is inherently singlethreaded (aside from CRC calculation) how can it be more than 100% faster? Because pgzip due to its design also acts as a buffer. When using unbuffered gzip, you are also waiting for io when you are decompressing. If the gzip decoder can keep up, it will always have data ready for your reader, and you will not be waiting for input to the gzip decompressor to complete.
	130
	131	This is pretty much an optimal situation for pgzip, but it reflects most common usecases for CPU intensive gzip usage.
	132
	133	I haven't included [bgzf](https://godoc.org/github.com/biogo/hts/bgzf) in this comparison, since it only can decompress files created by a compatible encoder, and therefore cannot be considered a generic gzip decompressor. But if you are able to compress your files with a bgzf compatible program, you can expect it to scale beyond 100%.
	134
	135	# License
	136	This contains large portions of code from the go repository - see GO_LICENSE for more information. The changes are released under MIT License. See LICENSE for more information.

vendor/github.com/klauspost/pgzip/circle.yml+7−0

@@ -0,0 +1,7 @@
	1	test:
	2	pre:
	3	- go vet ./...
	4
	5	override:
	6	- go test -v -cpu=1,2,4 .
	7	- go test -v -cpu=2 -race -short .

vendor/github.com/klauspost/pgzip/gunzip.go+573−0

@@ -0,0 +1,573 @@
	1	// Copyright 2009 The Go Authors. All rights reserved.
	2	// Use of this source code is governed by a BSD-style
	3	// license that can be found in the LICENSE file.
	4
	5	// Package pgzip implements reading and writing of gzip format compressed files,
	6	// as specified in RFC 1952.
	7	//
	8	// This is a drop in replacement for "compress/gzip".
	9	// This will split compression into blocks that are compressed in parallel.
	10	// This can be useful for compressing big amounts of data.
	11	// The gzip decompression has not been modified, but remains in the package,
	12	// so you can use it as a complete replacement for "compress/gzip".
	13	//
	14	// See more at https://github.com/klauspost/pgzip
	15	package pgzip
	16
	17	import (
	18	"bufio"
	19	"errors"
	20	"hash"
	21	"io"
	22	"sync"
	23	"time"
	24
	25	"github.com/klauspost/compress/flate"
	26	"github.com/klauspost/crc32"
	27	)
	28
	29	const (
	30	gzipID1 = 0x1f
	31	gzipID2 = 0x8b
	32	gzipDeflate = 8
	33	flagText = 1 << 0
	34	flagHdrCrc = 1 << 1
	35	flagExtra = 1 << 2
	36	flagName = 1 << 3
	37	flagComment = 1 << 4
	38	)
	39
	40	func makeReader(r io.Reader) flate.Reader {
	41	if rr, ok := r.(flate.Reader); ok {
	42	return rr
	43	}
	44	return bufio.NewReader(r)
	45	}
	46
	47	var (
	48	// ErrChecksum is returned when reading GZIP data that has an invalid checksum.
	49	ErrChecksum = errors.New("gzip: invalid checksum")
	50	// ErrHeader is returned when reading GZIP data that has an invalid header.
	51	ErrHeader = errors.New("gzip: invalid header")
	52	)
	53
	54	// The gzip file stores a header giving metadata about the compressed file.
	55	// That header is exposed as the fields of the Writer and Reader structs.
	56	type Header struct {
	57	Comment string // comment
	58	Extra []byte // "extra data"
	59	ModTime time.Time // modification time
	60	Name string // file name
	61	OS byte // operating system type
	62	}
	63
	64	// A Reader is an io.Reader that can be read to retrieve
	65	// uncompressed data from a gzip-format compressed file.
	66	//
	67	// In general, a gzip file can be a concatenation of gzip files,
	68	// each with its own header. Reads from the Reader
	69	// return the concatenation of the uncompressed data of each.
	70	// Only the first header is recorded in the Reader fields.
	71	//
	72	// Gzip files store a length and checksum of the uncompressed data.
	73	// The Reader will return a ErrChecksum when Read
	74	// reaches the end of the uncompressed data if it does not
	75	// have the expected length or checksum. Clients should treat data
	76	// returned by Read as tentative until they receive the io.EOF
	77	// marking the end of the data.
	78	type Reader struct {
	79	Header
	80	r flate.Reader
	81	decompressor io.ReadCloser
	82	digest hash.Hash32
	83	size uint32
	84	flg byte
	85	buf [512]byte
	86	err error
	87	closeErr chan error
	88	multistream bool
	89
	90	readAhead chan read
	91	roff int // read offset
	92	current []byte
	93	closeReader chan struct{}
	94	lastBlock bool
	95	blockSize int
	96	blocks int
	97
	98	activeRA bool // Indication if readahead is active
	99	mu sync.Mutex // Lock for above
	100
	101	blockPool chan []byte
	102	}
	103
	104	type read struct {
	105	b []byte
	106	err error
	107	}
	108
	109	// NewReader creates a new Reader reading the given reader.
	110	// The implementation buffers input and may read more data than necessary from r.
	111	// It is the caller's responsibility to call Close on the Reader when done.
	112	func NewReader(r io.Reader) (*Reader, error) {
	113	z := new(Reader)
	114	z.blocks = defaultBlocks
	115	z.blockSize = defaultBlockSize
	116	z.r = makeReader(r)
	117	z.digest = crc32.NewIEEE()
	118	z.multistream = true
	119	z.blockPool = make(chan []byte, z.blocks)
	120	for i := 0; i < z.blocks; i++ {
	121	z.blockPool <- make([]byte, z.blockSize)
	122	}
	123	if err := z.readHeader(true); err != nil {
	124	return nil, err
	125	}
	126	return z, nil
	127	}
	128
	129	// NewReaderN creates a new Reader reading the given reader.
	130	// The implementation buffers input and may read more data than necessary from r.
	131	// It is the caller's responsibility to call Close on the Reader when done.
	132	//
	133	// With this you can control the approximate size of your blocks,
	134	// as well as how many blocks you want to have prefetched.
	135	//
	136	// Default values for this is blockSize = 250000, blocks = 16,
	137	// meaning up to 16 blocks of maximum 250000 bytes will be
	138	// prefetched.
	139	func NewReaderN(r io.Reader, blockSize, blocks int) (*Reader, error) {
	140	z := new(Reader)
	141	z.blocks = blocks
	142	z.blockSize = blockSize
	143	z.r = makeReader(r)
	144	z.digest = crc32.NewIEEE()
	145	z.multistream = true
	146
	147	// Account for too small values
	148	if z.blocks <= 0 {
	149	z.blocks = defaultBlocks
	150	}
	151	if z.blockSize <= 512 {
	152	z.blockSize = defaultBlockSize
	153	}
	154	z.blockPool = make(chan []byte, z.blocks)
	155	for i := 0; i < z.blocks; i++ {
	156	z.blockPool <- make([]byte, z.blockSize)
	157	}
	158	if err := z.readHeader(true); err != nil {
	159	return nil, err
	160	}
	161	return z, nil
	162	}
	163
	164	// Reset discards the Reader z's state and makes it equivalent to the
	165	// result of its original state from NewReader, but reading from r instead.
	166	// This permits reusing a Reader rather than allocating a new one.
	167	func (z *Reader) Reset(r io.Reader) error {
	168	z.killReadAhead()
	169	z.r = makeReader(r)
	170	z.digest = crc32.NewIEEE()
	171	z.size = 0
	172	z.err = nil
	173	z.multistream = true
	174
	175	// Account for uninitialized values
	176	if z.blocks <= 0 {
	177	z.blocks = defaultBlocks
	178	}
	179	if z.blockSize <= 512 {
	180	z.blockSize = defaultBlockSize
	181	}
	182
	183	if z.blockPool == nil {
	184	z.blockPool = make(chan []byte, z.blocks)
	185	for i := 0; i < z.blocks; i++ {
	186	z.blockPool <- make([]byte, z.blockSize)
	187	}
	188	}
	189
	190	return z.readHeader(true)
	191	}
	192
	193	// Multistream controls whether the reader supports multistream files.
	194	//
	195	// If enabled (the default), the Reader expects the input to be a sequence
	196	// of individually gzipped data streams, each with its own header and
	197	// trailer, ending at EOF. The effect is that the concatenation of a sequence
	198	// of gzipped files is treated as equivalent to the gzip of the concatenation
	199	// of the sequence. This is standard behavior for gzip readers.
	200	//
	201	// Calling Multistream(false) disables this behavior; disabling the behavior
	202	// can be useful when reading file formats that distinguish individual gzip
	203	// data streams or mix gzip data streams with other data streams.
	204	// In this mode, when the Reader reaches the end of the data stream,
	205	// Read returns io.EOF. If the underlying reader implements io.ByteReader,
	206	// it will be left positioned just after the gzip stream.
	207	// To start the next stream, call z.Reset(r) followed by z.Multistream(false).
	208	// If there is no next stream, z.Reset(r) will return io.EOF.
	209	func (z *Reader) Multistream(ok bool) {
	210	z.multistream = ok
	211	}
	212
	213	// GZIP (RFC 1952) is little-endian, unlike ZLIB (RFC 1950).
	214	func get4(p []byte) uint32 {
	215	return uint32(p[0]) \| uint32(p[1])<<8 \| uint32(p[2])<<16 \| uint32(p[3])<<24
	216	}
	217
	218	func (z *Reader) readString() (string, error) {
	219	var err error
	220	needconv := false
	221	for i := 0; ; i++ {
	222	if i >= len(z.buf) {
	223	return "", ErrHeader
	224	}
	225	z.buf[i], err = z.r.ReadByte()
	226	if err != nil {
	227	return "", err
	228	}
	229	if z.buf[i] > 0x7f {
	230	needconv = true
	231	}
	232	if z.buf[i] == 0 {
	233	// GZIP (RFC 1952) specifies that strings are NUL-terminated ISO 8859-1 (Latin-1).
	234	if needconv {
	235	s := make([]rune, 0, i)
	236	for _, v := range z.buf[0:i] {
	237	s = append(s, rune(v))
	238	}
	239	return string(s), nil
	240	}
	241	return string(z.buf[0:i]), nil
	242	}
	243	}
	244	}
	245
	246	func (z *Reader) read2() (uint32, error) {
	247	_, err := io.ReadFull(z.r, z.buf[0:2])
	248	if err != nil {
	249	return 0, err
	250	}
	251	return uint32(z.buf[0]) \| uint32(z.buf[1])<<8, nil
	252	}
	253
	254	func (z *Reader) readHeader(save bool) error {
	255	z.killReadAhead()
	256
	257	_, err := io.ReadFull(z.r, z.buf[0:10])
	258	if err != nil {
	259	return err
	260	}
	261	if z.buf[0] != gzipID1 \|\| z.buf[1] != gzipID2 \|\| z.buf[2] != gzipDeflate {
	262	return ErrHeader
	263	}
	264	z.flg = z.buf[3]
	265	if save {
	266	z.ModTime = time.Unix(int64(get4(z.buf[4:8])), 0)
	267	// z.buf[8] is xfl, ignored
	268	z.OS = z.buf[9]
	269	}
	270	z.digest.Reset()
	271	z.digest.Write(z.buf[0:10])
	272
	273	if z.flg&flagExtra != 0 {
	274	n, err := z.read2()
	275	if err != nil {
	276	return err
	277	}
	278	data := make([]byte, n)
	279	if _, err = io.ReadFull(z.r, data); err != nil {
	280	return err
	281	}
	282	if save {
	283	z.Extra = data
	284	}
	285	}
	286
	287	var s string
	288	if z.flg&flagName != 0 {
	289	if s, err = z.readString(); err != nil {
	290	return err
	291	}
	292	if save {
	293	z.Name = s
	294	}
	295	}
	296
	297	if z.flg&flagComment != 0 {
	298	if s, err = z.readString(); err != nil {
	299	return err
	300	}
	301	if save {
	302	z.Comment = s
	303	}
	304	}
	305
	306	if z.flg&flagHdrCrc != 0 {
	307	n, err := z.read2()
	308	if err != nil {
	309	return err
	310	}
	311	sum := z.digest.Sum32() & 0xFFFF
	312	if n != sum {
	313	return ErrHeader
	314	}
	315	}
	316
	317	z.digest.Reset()
	318	z.decompressor = flate.NewReader(z.r)
	319	z.doReadAhead()
	320	return nil
	321	}
	322
	323	func (z *Reader) killReadAhead() error {
	324	z.mu.Lock()
	325	defer z.mu.Unlock()
	326	if z.activeRA {
	327	if z.closeReader != nil {
	328	close(z.closeReader)
	329	}
	330
	331	// Wait for decompressor to be closed and return error, if any.
	332	e, ok := <-z.closeErr
	333	z.activeRA = false
	334	if !ok {
	335	// Channel is closed, so if there was any error it has already been returned.
	336	return nil
	337	}
	338	return e
	339	}
	340	return nil
	341	}
	342
	343	// Starts readahead.
	344	// Will return on error (including io.EOF)
	345	// or when z.closeReader is closed.
	346	func (z *Reader) doReadAhead() {
	347	z.mu.Lock()
	348	defer z.mu.Unlock()
	349	z.activeRA = true
	350
	351	if z.blocks <= 0 {
	352	z.blocks = defaultBlocks
	353	}
	354	if z.blockSize <= 512 {
	355	z.blockSize = defaultBlockSize
	356	}
	357	ra := make(chan read, z.blocks)
	358	z.readAhead = ra
	359	closeReader := make(chan struct{}, 0)
	360	z.closeReader = closeReader
	361	z.lastBlock = false
	362	closeErr := make(chan error, 1)
	363	z.closeErr = closeErr
	364	z.size = 0
	365	z.roff = 0
	366	z.current = nil
	367	decomp := z.decompressor
	368
	369	go func() {
	370	defer func() {
	371	closeErr <- decomp.Close()
	372	close(closeErr)
	373	close(ra)
	374	}()
	375
	376	// We hold a local reference to digest, since
	377	// it way be changed by reset.
	378	digest := z.digest
	379	var wg sync.WaitGroup
	380	for {
	381	var buf []byte
	382	select {
	383	case buf = <-z.blockPool:
	384	case <-closeReader:
	385	return
	386	}
	387	buf = buf[0:z.blockSize]
	388	// Try to fill the buffer
	389	n, err := io.ReadFull(decomp, buf)
	390	if err == io.ErrUnexpectedEOF {
	391	if n > 0 {
	392	err = nil
	393	} else {
	394	// If we got zero bytes, we need to establish if
	395	// we reached end of stream or truncated stream.
	396	_, err = decomp.Read([]byte{})
	397	if err == io.EOF {
	398	err = nil
	399	}
	400	}
	401	}
	402	if n < len(buf) {
	403	buf = buf[0:n]
	404	}
	405	wg.Wait()
	406	wg.Add(1)
	407	go func() {
	408	digest.Write(buf)
	409	wg.Done()
	410	}()
	411	z.size += uint32(n)
	412
	413	// If we return any error, out digest must be ready
	414	if err != nil {
	415	wg.Wait()
	416	}
	417	select {
	418	case z.readAhead <- read{b: buf, err: err}:
	419	case <-closeReader:
	420	// Sent on close, we don't care about the next results
	421	return
	422	}
	423	if err != nil {
	424	return
	425	}
	426	}
	427	}()
	428	}
	429
	430	func (z *Reader) Read(p []byte) (n int, err error) {
	431	if z.err != nil {
	432	return 0, z.err
	433	}
	434	if len(p) == 0 {
	435	return 0, nil
	436	}
	437
	438	for {
	439	if len(z.current) == 0 && !z.lastBlock {
	440	read := <-z.readAhead
	441
	442	if read.err != nil {
	443	// If not nil, the reader will have exited
	444	z.closeReader = nil
	445
	446	if read.err != io.EOF {
	447	z.err = read.err
	448	return
	449	}
	450	if read.err == io.EOF {
	451	z.lastBlock = true
	452	err = nil
	453	}
	454	}
	455	z.current = read.b
	456	z.roff = 0
	457	}
	458	avail := z.current[z.roff:]
	459	if len(p) >= len(avail) {
	460	// If len(p) >= len(current), return all content of current
	461	n = copy(p, avail)
	462	z.blockPool <- z.current
	463	z.current = nil
	464	if z.lastBlock {
	465	err = io.EOF
	466	break
	467	}
	468	} else {
	469	// We copy as much as there is space for
	470	n = copy(p, avail)
	471	z.roff += n
	472	}
	473	return
	474	}
	475
	476	// Finished file; check checksum + size.
	477	if _, err := io.ReadFull(z.r, z.buf[0:8]); err != nil {
	478	z.err = err
	479	return 0, err
	480	}
	481	crc32, isize := get4(z.buf[0:4]), get4(z.buf[4:8])
	482	sum := z.digest.Sum32()
	483	if sum != crc32 \|\| isize != z.size {
	484	z.err = ErrChecksum
	485	return 0, z.err
	486	}
	487
	488	// File is ok; should we attempt reading one more?
	489	if !z.multistream {
	490	return 0, io.EOF
	491	}
	492
	493	// Is there another?
	494	if err = z.readHeader(false); err != nil {
	495	z.err = err
	496	return
	497	}
	498
	499	// Yes. Reset and read from it.
	500	return z.Read(p)
	501	}
	502
	503	func (z *Reader) WriteTo(w io.Writer) (n int64, err error) {
	504	total := int64(0)
	505	for {
	506	if z.err != nil {
	507	return total, z.err
	508	}
	509	// We write both to output and digest.
	510	for {
	511	// Read from input
	512	read := <-z.readAhead
	513	if read.err != nil {
	514	// If not nil, the reader will have exited
	515	z.closeReader = nil
	516
	517	if read.err != io.EOF {
	518	z.err = read.err
	519	return total, z.err
	520	}
	521	if read.err == io.EOF {
	522	z.lastBlock = true
	523	err = nil
	524	}
	525	}
	526	// Write what we got
	527	n, err := w.Write(read.b)
	528	if n != len(read.b) {
	529	return total, io.ErrShortWrite
	530	}
	531	total += int64(n)
	532	if err != nil {
	533	return total, err
	534	}
	535	// Put block back
	536	z.blockPool <- read.b
	537	if z.lastBlock {
	538	break
	539	}
	540	}
	541
	542	// Finished file; check checksum + size.
	543	if _, err := io.ReadFull(z.r, z.buf[0:8]); err != nil {
	544	z.err = err
	545	return total, err
	546	}
	547	crc32, isize := get4(z.buf[0:4]), get4(z.buf[4:8])
	548	sum := z.digest.Sum32()
	549	if sum != crc32 \|\| isize != z.size {
	550	z.err = ErrChecksum
	551	return total, z.err
	552	}
	553	// File is ok; should we attempt reading one more?
	554	if !z.multistream {
	555	return total, nil
	556	}
	557
	558	// Is there another?
	559	err = z.readHeader(false)
	560	if err == io.EOF {
	561	return total, nil
	562	}
	563	if err != nil {
	564	z.err = err
	565	return total, err
	566	}
	567	}
	568	}
	569
	570	// Close closes the Reader. It does not close the underlying io.Reader.
	571	func (z *Reader) Close() error {
	572	return z.killReadAhead()
	573	}

vendor/github.com/klauspost/pgzip/gzip.go+501−0

@@ -0,0 +1,501 @@
	1	// Copyright 2010 The Go Authors. All rights reserved.
	2	// Use of this source code is governed by a BSD-style
	3	// license that can be found in the LICENSE file.
	4
	5	package pgzip
	6
	7	import (
	8	"bytes"
	9	"errors"
	10	"fmt"
	11	"hash"
	12	"io"
	13	"sync"
	14	"time"
	15
	16	"github.com/klauspost/compress/flate"
	17	"github.com/klauspost/crc32"
	18	)
	19
	20	const (
	21	defaultBlockSize = 256 << 10
	22	tailSize = 16384
	23	defaultBlocks = 16
	24	)
	25
	26	// These constants are copied from the flate package, so that code that imports
	27	// "compress/gzip" does not also have to import "compress/flate".
	28	const (
	29	NoCompression = flate.NoCompression
	30	BestSpeed = flate.BestSpeed
	31	BestCompression = flate.BestCompression
	32	DefaultCompression = flate.DefaultCompression
	33	ConstantCompression = flate.ConstantCompression
	34	HuffmanOnly = flate.HuffmanOnly
	35	)
	36
	37	// A Writer is an io.WriteCloser.
	38	// Writes to a Writer are compressed and written to w.
	39	type Writer struct {
	40	Header
	41	w io.Writer
	42	level int
	43	wroteHeader bool
	44	blockSize int
	45	blocks int
	46	currentBuffer []byte
	47	prevTail []byte
	48	digest hash.Hash32
	49	size int
	50	closed bool
	51	buf [10]byte
	52	errMu sync.RWMutex
	53	err error
	54	pushedErr chan struct{}
	55	results chan result
	56	dictFlatePool sync.Pool
	57	dstPool sync.Pool
	58	wg sync.WaitGroup
	59	}
	60
	61	type result struct {
	62	result chan []byte
	63	notifyWritten chan struct{}
	64	}
	65
	66	// Use SetConcurrency to finetune the concurrency level if needed.
	67	//
	68	// With this you can control the approximate size of your blocks,
	69	// as well as how many you want to be processing in parallel.
	70	//
	71	// Default values for this is SetConcurrency(250000, 16),
	72	// meaning blocks are split at 250000 bytes and up to 16 blocks
	73	// can be processing at once before the writer blocks.
	74	func (z *Writer) SetConcurrency(blockSize, blocks int) error {
	75	if blockSize <= tailSize {
	76	return fmt.Errorf("gzip: block size cannot be less than or equal to %d", tailSize)
	77	}
	78	if blocks <= 0 {
	79	return errors.New("gzip: blocks cannot be zero or less")
	80	}
	81	if blockSize == z.blockSize && blocks == z.blocks {
	82	return nil
	83	}
	84	z.blockSize = blockSize
	85	z.results = make(chan result, blocks)
	86	z.blocks = blocks
	87	z.dstPool = sync.Pool{New: func() interface{} { return make([]byte, 0, blockSize+(blockSize)>>4) }}
	88	return nil
	89	}
	90
	91	// NewWriter returns a new Writer.
	92	// Writes to the returned writer are compressed and written to w.
	93	//
	94	// It is the caller's responsibility to call Close on the WriteCloser when done.
	95	// Writes may be buffered and not flushed until Close.
	96	//
	97	// Callers that wish to set the fields in Writer.Header must do so before
	98	// the first call to Write or Close. The Comment and Name header fields are
	99	// UTF-8 strings in Go, but the underlying format requires NUL-terminated ISO
	100	// 8859-1 (Latin-1). NUL or non-Latin-1 runes in those strings will lead to an
	101	// error on Write.
	102	func NewWriter(w io.Writer) *Writer {
	103	z, _ := NewWriterLevel(w, DefaultCompression)
	104	return z
	105	}
	106
	107	// NewWriterLevel is like NewWriter but specifies the compression level instead
	108	// of assuming DefaultCompression.
	109	//
	110	// The compression level can be DefaultCompression, NoCompression, or any
	111	// integer value between BestSpeed and BestCompression inclusive. The error
	112	// returned will be nil if the level is valid.
	113	func NewWriterLevel(w io.Writer, level int) (*Writer, error) {
	114	if level < ConstantCompression \|\| level > BestCompression {
	115	return nil, fmt.Errorf("gzip: invalid compression level: %d", level)
	116	}
	117	z := new(Writer)
	118	z.SetConcurrency(defaultBlockSize, defaultBlocks)
	119	z.init(w, level)
	120	return z, nil
	121	}
	122
	123	// This function must be used by goroutines to set an
	124	// error condition, since z.err access is restricted
	125	// to the callers goruotine.
	126	func (z *Writer) pushError(err error) {
	127	z.errMu.Lock()
	128	if z.err != nil {
	129	z.errMu.Unlock()
	130	return
	131	}
	132	z.err = err
	133	close(z.pushedErr)
	134	z.errMu.Unlock()
	135	}
	136
	137	func (z *Writer) init(w io.Writer, level int) {
	138	z.wg.Wait()
	139	digest := z.digest
	140	if digest != nil {
	141	digest.Reset()
	142	} else {
	143	digest = crc32.NewIEEE()
	144	}
	145	z.Header = Header{OS: 255}
	146	z.w = w
	147	z.level = level
	148	z.digest = digest
	149	z.pushedErr = make(chan struct{}, 0)
	150	z.results = make(chan result, z.blocks)
	151	z.err = nil
	152	z.closed = false
	153	z.Comment = ""
	154	z.Extra = nil
	155	z.ModTime = time.Time{}
	156	z.wroteHeader = false
	157	z.currentBuffer = nil
	158	z.buf = [10]byte{}
	159	z.prevTail = nil
	160	z.size = 0
	161	if z.dictFlatePool.New == nil {
	162	z.dictFlatePool.New = func() interface{} {
	163	f, _ := flate.NewWriterDict(w, level, nil)
	164	return f
	165	}
	166	}
	167	}
	168
	169	// Reset discards the Writer z's state and makes it equivalent to the
	170	// result of its original state from NewWriter or NewWriterLevel, but
	171	// writing to w instead. This permits reusing a Writer rather than
	172	// allocating a new one.
	173	func (z *Writer) Reset(w io.Writer) {
	174	if z.results != nil && !z.closed {
	175	close(z.results)
	176	}
	177	z.SetConcurrency(defaultBlockSize, defaultBlocks)
	178	z.init(w, z.level)
	179	}
	180
	181	// GZIP (RFC 1952) is little-endian, unlike ZLIB (RFC 1950).
	182	func put2(p []byte, v uint16) {
	183	p[0] = uint8(v >> 0)
	184	p[1] = uint8(v >> 8)
	185	}
	186
	187	func put4(p []byte, v uint32) {
	188	p[0] = uint8(v >> 0)
	189	p[1] = uint8(v >> 8)
	190	p[2] = uint8(v >> 16)
	191	p[3] = uint8(v >> 24)
	192	}
	193
	194	// writeBytes writes a length-prefixed byte slice to z.w.
	195	func (z *Writer) writeBytes(b []byte) error {
	196	if len(b) > 0xffff {
	197	return errors.New("gzip.Write: Extra data is too large")
	198	}
	199	put2(z.buf[0:2], uint16(len(b)))
	200	_, err := z.w.Write(z.buf[0:2])
	201	if err != nil {
	202	return err
	203	}
	204	_, err = z.w.Write(b)
	205	return err
	206	}
	207
	208	// writeString writes a UTF-8 string s in GZIP's format to z.w.
	209	// GZIP (RFC 1952) specifies that strings are NUL-terminated ISO 8859-1 (Latin-1).
	210	func (z *Writer) writeString(s string) (err error) {
	211	// GZIP stores Latin-1 strings; error if non-Latin-1; convert if non-ASCII.
	212	needconv := false
	213	for _, v := range s {
	214	if v == 0 \|\| v > 0xff {
	215	return errors.New("gzip.Write: non-Latin-1 header string")
	216	}
	217	if v > 0x7f {
	218	needconv = true
	219	}
	220	}
	221	if needconv {
	222	b := make([]byte, 0, len(s))
	223	for _, v := range s {
	224	b = append(b, byte(v))
	225	}
	226	_, err = z.w.Write(b)
	227	} else {
	228	_, err = io.WriteString(z.w, s)
	229	}
	230	if err != nil {
	231	return err
	232	}
	233	// GZIP strings are NUL-terminated.
	234	z.buf[0] = 0
	235	_, err = z.w.Write(z.buf[0:1])
	236	return err
	237	}
	238
	239	// compressCurrent will compress the data currently buffered
	240	// This should only be called from the main writer/flush/closer
	241	func (z *Writer) compressCurrent(flush bool) {
	242	r := result{}
	243	r.result = make(chan []byte, 1)
	244	r.notifyWritten = make(chan struct{}, 0)
	245	select {
	246	case z.results <- r:
	247	case <-z.pushedErr:
	248	return
	249	}
	250
	251	// If block given is more than twice the block size, split it.
	252	c := z.currentBuffer
	253	if len(c) > z.blockSize*2 {
	254	c = c[:z.blockSize]
	255	z.wg.Add(1)
	256	go z.compressBlock(c, z.prevTail, r, false)
	257	z.prevTail = c[len(c)-tailSize:]
	258	z.currentBuffer = z.currentBuffer[z.blockSize:]
	259	z.compressCurrent(flush)
	260	// Last one flushes if needed
	261	return
	262	}
	263
	264	z.wg.Add(1)
	265	go z.compressBlock(c, z.prevTail, r, z.closed)
	266	if len(c) > tailSize {
	267	z.prevTail = c[len(c)-tailSize:]
	268	} else {
	269	z.prevTail = nil
	270	}
	271	z.currentBuffer = z.dstPool.Get().([]byte)
	272	z.currentBuffer = z.currentBuffer[:0]
	273
	274	// Wait if flushing
	275	if flush {
	276	<-r.notifyWritten
	277	}
	278	}
	279
	280	// Returns an error if it has been set.
	281	// Cannot be used by functions that are from internal goroutines.
	282	func (z *Writer) checkError() error {
	283	z.errMu.RLock()
	284	err := z.err
	285	z.errMu.RUnlock()
	286	return err
	287	}
	288
	289	// Write writes a compressed form of p to the underlying io.Writer. The
	290	// compressed bytes are not necessarily flushed to output until
	291	// the Writer is closed or Flush() is called.
	292	//
	293	// The function will return quickly, if there are unused buffers.
	294	// The sent slice (p) is copied, and the caller is free to re-use the buffer
	295	// when the function returns.
	296	//
	297	// Errors that occur during compression will be reported later, and a nil error
	298	// does not signify that the compression succeeded (since it is most likely still running)
	299	// That means that the call that returns an error may not be the call that caused it.
	300	// Only Flush and Close functions are guaranteed to return any errors up to that point.
	301	func (z *Writer) Write(p []byte) (int, error) {
	302	if err := z.checkError(); err != nil {
	303	return 0, err
	304	}
	305	// Write the GZIP header lazily.
	306	if !z.wroteHeader {
	307	z.wroteHeader = true
	308	z.buf[0] = gzipID1
	309	z.buf[1] = gzipID2
	310	z.buf[2] = gzipDeflate
	311	z.buf[3] = 0
	312	if z.Extra != nil {
	313	z.buf[3] \|= 0x04
	314	}
	315	if z.Name != "" {
	316	z.buf[3] \|= 0x08
	317	}
	318	if z.Comment != "" {
	319	z.buf[3] \|= 0x10
	320	}
	321	put4(z.buf[4:8], uint32(z.ModTime.Unix()))
	322	if z.level == BestCompression {
	323	z.buf[8] = 2
	324	} else if z.level == BestSpeed {
	325	z.buf[8] = 4
	326	} else {
	327	z.buf[8] = 0
	328	}
	329	z.buf[9] = z.OS
	330	var n int
	331	var err error
	332	n, err = z.w.Write(z.buf[0:10])
	333	if err != nil {
	334	z.pushError(err)
	335	return n, err
	336	}
	337	if z.Extra != nil {
	338	err = z.writeBytes(z.Extra)
	339	if err != nil {
	340	z.pushError(err)
	341	return n, err
	342	}
	343	}
	344	if z.Name != "" {
	345	err = z.writeString(z.Name)
	346	if err != nil {
	347	z.pushError(err)
	348	return n, err
	349	}
	350	}
	351	if z.Comment != "" {
	352	err = z.writeString(z.Comment)
	353	if err != nil {
	354	z.pushError(err)
	355	return n, err
	356	}
	357	}
	358	// Start receiving data from compressors
	359	go func() {
	360	listen := z.results
	361	for {
	362	r, ok := <-listen
	363	// If closed, we are finished.
	364	if !ok {
	365	return
	366	}
	367	buf := <-r.result
	368	n, err := z.w.Write(buf)
	369	if err != nil {
	370	z.pushError(err)
	371	close(r.notifyWritten)
	372	return
	373	}
	374	if n != len(buf) {
	375	z.pushError(fmt.Errorf("gzip: short write %d should be %d", n, len(buf)))
	376	close(r.notifyWritten)
	377	return
	378	}
	379	z.dstPool.Put(buf)
	380	close(r.notifyWritten)
	381	}
	382	}()
	383	z.currentBuffer = make([]byte, 0, z.blockSize)
	384	}
	385	q := p
	386	for len(q) > 0 {
	387	length := len(q)
	388	if length+len(z.currentBuffer) > z.blockSize {
	389	length = z.blockSize - len(z.currentBuffer)
	390	}
	391	z.digest.Write(q[:length])
	392	z.currentBuffer = append(z.currentBuffer, q[:length]...)
	393	if len(z.currentBuffer) >= z.blockSize {
	394	z.compressCurrent(false)
	395	if err := z.checkError(); err != nil {
	396	return len(p) - len(q) - length, err
	397	}
	398	}
	399	z.size += length
	400	q = q[length:]
	401	}
	402	return len(p), z.checkError()
	403	}
	404
	405	// Step 1: compresses buffer to buffer
	406	// Step 2: send writer to channel
	407	// Step 3: Close result channel to indicate we are done
	408	func (z *Writer) compressBlock(p, prevTail []byte, r result, closed bool) {
	409	defer func() {
	410	close(r.result)
	411	z.wg.Done()
	412	}()
	413	buf := z.dstPool.Get().([]byte)
	414	dest := bytes.NewBuffer(buf[:0])
	415
	416	compressor := z.dictFlatePool.Get().(*flate.Writer)
	417	compressor.ResetDict(dest, prevTail)
	418	compressor.Write(p)
	419
	420	err := compressor.Flush()
	421	if err != nil {
	422	z.pushError(err)
	423	return
	424	}
	425	if closed {
	426	err = compressor.Close()
	427	if err != nil {
	428	z.pushError(err)
	429	return
	430	}
	431	}
	432	z.dictFlatePool.Put(compressor)
	433	// Read back buffer
	434	buf = dest.Bytes()
	435	r.result <- buf
	436	}
	437
	438	// Flush flushes any pending compressed data to the underlying writer.
	439	//
	440	// It is useful mainly in compressed network protocols, to ensure that
	441	// a remote reader has enough data to reconstruct a packet. Flush does
	442	// not return until the data has been written. If the underlying
	443	// writer returns an error, Flush returns that error.
	444	//
	445	// In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH.
	446	func (z *Writer) Flush() error {
	447	if err := z.checkError(); err != nil {
	448	return err
	449	}
	450	if z.closed {
	451	return nil
	452	}
	453	if !z.wroteHeader {
	454	_, err := z.Write(nil)
	455	if err != nil {
	456	return err
	457	}
	458	}
	459	// We send current block to compression
	460	z.compressCurrent(true)
	461
	462	return z.checkError()
	463	}
	464
	465	// UncompressedSize will return the number of bytes written.
	466	// pgzip only, not a function in the official gzip package.
	467	func (z *Writer) UncompressedSize() int {
	468	return z.size
	469	}
	470
	471	// Close closes the Writer, flushing any unwritten data to the underlying
	472	// io.Writer, but does not close the underlying io.Writer.
	473	func (z *Writer) Close() error {
	474	if err := z.checkError(); err != nil {
	475	return err
	476	}
	477	if z.closed {
	478	return nil
	479	}
	480
	481	z.closed = true
	482	if !z.wroteHeader {
	483	z.Write(nil)
	484	if err := z.checkError(); err != nil {
	485	return err
	486	}
	487	}
	488	z.compressCurrent(true)
	489	if err := z.checkError(); err != nil {
	490	return err
	491	}
	492	close(z.results)
	493	put4(z.buf[0:4], z.digest.Sum32())
	494	put4(z.buf[4:8], uint32(z.size))
	495	_, err := z.w.Write(z.buf[0:8])
	496	if err != nil {
	497	z.pushError(err)
	498	return err
	499	}
	500	return nil
	501	}

vendor/github.com/xwb1989/sqlparser/CONTRIBUTORS.md+9−0

@@ -0,0 +1,9 @@
	1	This project is originally a fork of [https://github.com/youtube/vitess](https://github.com/youtube/vitess)
	2	Copyright Google Inc
	3
	4	# Contributors
	5	Wenbin Xiao 2015
	6	Started this project and maintained it.
	7
	8	Andrew Brampton 2017
	9	Merged in multiple upstream fixes/changes.

vendor/github.com/xwb1989/sqlparser/LICENSE.md+201−0

@@ -0,0 +1,201 @@
	1	Apache License
	2	Version 2.0, January 2004
	3	http://www.apache.org/licenses/
	4
	5	TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
	6
	7	1. Definitions.
	8
	9	"License" shall mean the terms and conditions for use, reproduction,
	10	and distribution as defined by Sections 1 through 9 of this document.
	11
	12	"Licensor" shall mean the copyright owner or entity authorized by
	13	the copyright owner that is granting the License.
	14
	15	"Legal Entity" shall mean the union of the acting entity and all
	16	other entities that control, are controlled by, or are under common
	17	control with that entity. For the purposes of this definition,
	18	"control" means (i) the power, direct or indirect, to cause the
	19	direction or management of such entity, whether by contract or
	20	otherwise, or (ii) ownership of fifty percent (50%) or more of the
	21	outstanding shares, or (iii) beneficial ownership of such entity.
	22
	23	"You" (or "Your") shall mean an individual or Legal Entity
	24	exercising permissions granted by this License.
	25
	26	"Source" form shall mean the preferred form for making modifications,
	27	including but not limited to software source code, documentation
	28	source, and configuration files.
	29
	30	"Object" form shall mean any form resulting from mechanical
	31	transformation or translation of a Source form, including but
	32	not limited to compiled object code, generated documentation,
	33	and conversions to other media types.
	34
	35	"Work" shall mean the work of authorship, whether in Source or
	36	Object form, made available under the License, as indicated by a
	37	copyright notice that is included in or attached to the work
	38	(an example is provided in the Appendix below).
	39
	40	"Derivative Works" shall mean any work, whether in Source or Object
	41	form, that is based on (or derived from) the Work and for which the
	42	editorial revisions, annotations, elaborations, or other modifications
	43	represent, as a whole, an original work of authorship. For the purposes
	44	of this License, Derivative Works shall not include works that remain
	45	separable from, or merely link (or bind by name) to the interfaces of,
	46	the Work and Derivative Works thereof.
	47
	48	"Contribution" shall mean any work of authorship, including
	49	the original version of the Work and any modifications or additions
	50	to that Work or Derivative Works thereof, that is intentionally
	51	submitted to Licensor for inclusion in the Work by the copyright owner
	52	or by an individual or Legal Entity authorized to submit on behalf of
	53	the copyright owner. For the purposes of this definition, "submitted"
	54	means any form of electronic, verbal, or written communication sent
	55	to the Licensor or its representatives, including but not limited to
	56	communication on electronic mailing lists, source code control systems,
	57	and issue tracking systems that are managed by, or on behalf of, the
	58	Licensor for the purpose of discussing and improving the Work, but
	59	excluding communication that is conspicuously marked or otherwise
	60	designated in writing by the copyright owner as "Not a Contribution."
	61
	62	"Contributor" shall mean Licensor and any individual or Legal Entity
	63	on behalf of whom a Contribution has been received by Licensor and
	64	subsequently incorporated within the Work.
	65
	66	2. Grant of Copyright License. Subject to the terms and conditions of
	67	this License, each Contributor hereby grants to You a perpetual,
	68	worldwide, non-exclusive, no-charge, royalty-free, irrevocable
	69	copyright license to reproduce, prepare Derivative Works of,
	70	publicly display, publicly perform, sublicense, and distribute the
	71	Work and such Derivative Works in Source or Object form.
	72
	73	3. Grant of Patent License. Subject to the terms and conditions of
	74	this License, each Contributor hereby grants to You a perpetual,
	75	worldwide, non-exclusive, no-charge, royalty-free, irrevocable
	76	(except as stated in this section) patent license to make, have made,
	77	use, offer to sell, sell, import, and otherwise transfer the Work,
	78	where such license applies only to those patent claims licensable
	79	by such Contributor that are necessarily infringed by their
	80	Contribution(s) alone or by combination of their Contribution(s)
	81	with the Work to which such Contribution(s) was submitted. If You
	82	institute patent litigation against any entity (including a
	83	cross-claim or counterclaim in a lawsuit) alleging that the Work
	84	or a Contribution incorporated within the Work constitutes direct
	85	or contributory patent infringement, then any patent licenses
	86	granted to You under this License for that Work shall terminate
	87	as of the date such litigation is filed.
	88
	89	4. Redistribution. You may reproduce and distribute copies of the
	90	Work or Derivative Works thereof in any medium, with or without
	91	modifications, and in Source or Object form, provided that You
	92	meet the following conditions:
	93
	94	(a) You must give any other recipients of the Work or
	95	Derivative Works a copy of this License; and
	96
	97	(b) You must cause any modified files to carry prominent notices
	98	stating that You changed the files; and
	99
	100	(c) You must retain, in the Source form of any Derivative Works
	101	that You distribute, all copyright, patent, trademark, and
	102	attribution notices from the Source form of the Work,
	103	excluding those notices that do not pertain to any part of
	104	the Derivative Works; and
	105
	106	(d) If the Work includes a "NOTICE" text file as part of its
	107	distribution, then any Derivative Works that You distribute must
	108	include a readable copy of the attribution notices contained
	109	within such NOTICE file, excluding those notices that do not
	110	pertain to any part of the Derivative Works, in at least one
	111	of the following places: within a NOTICE text file distributed
	112	as part of the Derivative Works; within the Source form or
	113	documentation, if provided along with the Derivative Works; or,
	114	within a display generated by the Derivative Works, if and
	115	wherever such third-party notices normally appear. The contents
	116	of the NOTICE file are for informational purposes only and
	117	do not modify the License. You may add Your own attribution
	118	notices within Derivative Works that You distribute, alongside
	119	or as an addendum to the NOTICE text from the Work, provided
	120	that such additional attribution notices cannot be construed
	121	as modifying the License.
	122
	123	You may add Your own copyright statement to Your modifications and
	124	may provide additional or different license terms and conditions
	125	for use, reproduction, or distribution of Your modifications, or
	126	for any such Derivative Works as a whole, provided Your use,
	127	reproduction, and distribution of the Work otherwise complies with
	128	the conditions stated in this License.
	129
	130	5. Submission of Contributions. Unless You explicitly state otherwise,
	131	any Contribution intentionally submitted for inclusion in the Work
	132	by You to the Licensor shall be under the terms and conditions of
	133	this License, without any additional terms or conditions.
	134	Notwithstanding the above, nothing herein shall supersede or modify
	135	the terms of any separate license agreement you may have executed
	136	with Licensor regarding such Contributions.
	137
	138	6. Trademarks. This License does not grant permission to use the trade
	139	names, trademarks, service marks, or product names of the Licensor,
	140	except as required for reasonable and customary use in describing the
	141	origin of the Work and reproducing the content of the NOTICE file.
	142
	143	7. Disclaimer of Warranty. Unless required by applicable law or
	144	agreed to in writing, Licensor provides the Work (and each
	145	Contributor provides its Contributions) on an "AS IS" BASIS,
	146	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
	147	implied, including, without limitation, any warranties or conditions
	148	of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
	149	PARTICULAR PURPOSE. You are solely responsible for determining the
	150	appropriateness of using or redistributing the Work and assume any
	151	risks associated with Your exercise of permissions under this License.
	152
	153	8. Limitation of Liability. In no event and under no legal theory,
	154	whether in tort (including negligence), contract, or otherwise,
	155	unless required by applicable law (such as deliberate and grossly
	156	negligent acts) or agreed to in writing, shall any Contributor be
	157	liable to You for damages, including any direct, indirect, special,
	158	incidental, or consequential damages of any character arising as a
	159	result of this License or out of the use or inability to use the
	160	Work (including but not limited to damages for loss of goodwill,
	161	work stoppage, computer failure or malfunction, or any and all
	162	other commercial damages or losses), even if such Contributor
	163	has been advised of the possibility of such damages.
	164
	165	9. Accepting Warranty or Additional Liability. While redistributing
	166	the Work or Derivative Works thereof, You may choose to offer,
	167	and charge a fee for, acceptance of support, warranty, indemnity,
	168	or other liability obligations and/or rights consistent with this
	169	License. However, in accepting such obligations, You may act only
	170	on Your own behalf and on Your sole responsibility, not on behalf
	171	of any other Contributor, and only if You agree to indemnify,
	172	defend, and hold each Contributor harmless for any liability
	173	incurred by, or claims asserted against, such Contributor by reason
	174	of your accepting any such warranty or additional liability.
	175
	176	END OF TERMS AND CONDITIONS
	177
	178	APPENDIX: How to apply the Apache License to your work.
	179
	180	To apply the Apache License to your work, attach the following
	181	boilerplate notice, with the fields enclosed by brackets "[]"
	182	replaced with your own identifying information. (Don't include
	183	the brackets!) The text should be enclosed in the appropriate
	184	comment syntax for the file format. We also recommend that a
	185	file or class name and description of purpose be included on the
	186	same "printed page" as the copyright notice for easier
	187	identification within third-party archives.
	188
	189	Copyright [yyyy] [name of copyright owner]
	190
	191	Licensed under the Apache License, Version 2.0 (the "License");
	192	you may not use this file except in compliance with the License.
	193	You may obtain a copy of the License at
	194
	195	http://www.apache.org/licenses/LICENSE-2.0
	196
	197	Unless required by applicable law or agreed to in writing, software
	198	distributed under the License is distributed on an "AS IS" BASIS,
	199	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	200	See the License for the specific language governing permissions and
	201	limitations under the License.

vendor/github.com/xwb1989/sqlparser/Makefile+22−0

@@ -0,0 +1,22 @@
	1	# Copyright 2017 Google Inc.
	2	#
	3	# Licensed under the Apache License, Version 2.0 (the "License");
	4	# you may not use this file except in compliance with the License.
	5	# You may obtain a copy of the License at
	6	#
	7	# http://www.apache.org/licenses/LICENSE-2.0
	8	#
	9	# Unless required by applicable law or agreed to in writing, software
	10	# distributed under the License is distributed on an "AS IS" BASIS,
	11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	# See the License for the specific language governing permissions and
	13	# limitations under the License.
	14
	15	MAKEFLAGS = -s
	16
	17	sql.go: sql.y
	18	goyacc -o sql.go sql.y
	19	gofmt -w sql.go
	20
	21	clean:
	22	rm -f y.output sql.go

vendor/github.com/xwb1989/sqlparser/README.md+150−0

@@ -0,0 +1,150 @@
	1	# sqlparser [![Build Status](https://img.shields.io/travis/xwb1989/sqlparser.svg)](https://travis-ci.org/xwb1989/sqlparser) [![Coverage](https://img.shields.io/coveralls/xwb1989/sqlparser.svg)](https://coveralls.io/github/xwb1989/sqlparser) [![Report card](https://goreportcard.com/badge/github.com/xwb1989/sqlparser)](https://goreportcard.com/report/github.com/xwb1989/sqlparser) [![GoDoc](https://godoc.org/github.com/xwb1989/sqlparser?status.svg)](https://godoc.org/github.com/xwb1989/sqlparser)
	2
	3	Go package for parsing MySQL SQL queries.
	4
	5	## Notice
	6
	7	The backbone of this repo is extracted from [vitessio/vitess](https://github.com/vitessio/vitess).
	8
	9	Inside vitessio/vitess there is a very nicely written sql parser. However as it's not a self-contained application, I created this one.
	10	It applies the same LICENSE as vitessio/vitess.
	11
	12	## Usage
	13
	14	```go
	15	import (
	16	"github.com/xwb1989/sqlparser"
	17	)
	18	```
	19
	20	Then use:
	21
	22	```go
	23	sql := "SELECT * FROM table WHERE a = 'abc'"
	24	stmt, err := sqlparser.Parse(sql)
	25	if err != nil {
	26	// Do something with the err
	27	}
	28
	29	// Otherwise do something with stmt
	30	switch stmt := stmt.(type) {
	31	case *sqlparser.Select:
	32	_ = stmt
	33	case *sqlparser.Insert:
	34	}
	35	```
	36
	37	Alternative to read many queries from a io.Reader:
	38
	39	```go
	40	r := strings.NewReader("INSERT INTO table1 VALUES (1, 'a'); INSERT INTO table2 VALUES (3, 4);")
	41
	42	tokens := sqlparser.NewTokenizer(r)
	43	for {
	44	stmt, err := sqlparser.ParseNext(tokens)
	45	if err == io.EOF {
	46	break
	47	}
	48	// Do something with stmt or err.
	49	}
	50	```
	51
	52	See [parse_test.go](https://github.com/xwb1989/sqlparser/blob/master/parse_test.go) for more examples, or read the [godoc](https://godoc.org/github.com/xwb1989/sqlparser).
	53
	54
	55	## Porting Instructions
	56
	57	You only need the below if you plan to try and keep this library up to date with [vitessio/vitess](https://github.com/vitessio/vitess).
	58
	59	### Keeping up to date
	60
	61	```bash
	62	shopt -s nullglob
	63	VITESS=${GOPATH?}/src/vitess.io/vitess/go/
	64	XWB1989=${GOPATH?}/src/github.com/xwb1989/sqlparser/
	65
	66	# Create patches for everything that changed
	67	LASTIMPORT=1b7879cb91f1dfe1a2dfa06fea96e951e3a7aec5
	68	for path in ${VITESS?}/{vt/sqlparser,sqltypes,bytes2,hack}; do
	69	cd ${path}
	70	git format-patch ${LASTIMPORT?} .
	71	done;
	72
	73	# Apply patches to the dependencies
	74	cd ${XWB1989?}
	75	git am --directory dependency -p2 ${VITESS?}/{sqltypes,bytes2,hack}/*.patch
	76
	77	# Apply the main patches to the repo
	78	cd ${XWB1989?}
	79	git am -p4 ${VITESS?}/vt/sqlparser/*.patch
	80
	81	# If you encounter diff failures, manually fix them with
	82	patch -p4 < .git/rebase-apply/patch
	83	...
	84	git add name_of_files
	85	git am --continue
	86
	87	# Cleanup
	88	rm ${VITESS?}/{sqltypes,bytes2,hack}/.patch ${VITESS?}/.patch
	89
	90	# and Finally update the LASTIMPORT in this README.
	91	```
	92
	93	### Fresh install
	94
	95	TODO: Change these instructions to use git to copy the files, that'll make later patching easier.
	96
	97	```bash
	98	VITESS=${GOPATH?}/src/vitess.io/vitess/go/
	99	XWB1989=${GOPATH?}/src/github.com/xwb1989/sqlparser/
	100
	101	cd ${XWB1989?}
	102
	103	# Copy all the code
	104	cp -pr ${VITESS?}/vt/sqlparser/ .
	105	cp -pr ${VITESS?}/sqltypes dependency
	106	cp -pr ${VITESS?}/bytes2 dependency
	107	cp -pr ${VITESS?}/hack dependency
	108
	109	# Delete some code we haven't ported
	110	rm dependency/sqltypes/arithmetic.go dependency/sqltypes/arithmetic_test.go dependency/sqltypes/event_token.go dependency/sqltypes/event_token_test.go dependency/sqltypes/proto3.go dependency/sqltypes/proto3_test.go dependency/sqltypes/query_response.go dependency/sqltypes/result.go dependency/sqltypes/result_test.go
	111
	112	# Some automated fixes
	113
	114	# Fix imports
	115	sed -i '.bak' 's_vitess.io/vitess/go/vt/proto/query_github.com/xwb1989/sqlparser/dependency/querypb_g' .go dependency/sqltypes/.go
	116	sed -i '.bak' 's_vitess.io/vitess/go/_github.com/xwb1989/sqlparser/dependency/_g' .go dependency/sqltypes/.go
	117
	118	# Copy the proto, but basically drop everything we don't want
	119	cp -pr ${VITESS?}/vt/proto/query dependency/querypb
	120
	121	sed -i '.bak' 's_.Descriptor.__g' dependency/querypb/*.go
	122	sed -i '.bak' 's_.ProtoMessage.__g' dependency/querypb/*.go
	123
	124	sed -i '.bak' 's/proto.CompactTextString(m)/"TODO"/g' dependency/querypb/*.go
	125	sed -i '.bak' 's/proto.EnumName/EnumName/g' dependency/querypb/*.go
	126
	127	sed -i '.bak' 's/proto.Equal/reflect.DeepEqual/g' dependency/sqltypes/*.go
	128
	129	# Remove the error library
	130	sed -i '.bak' 's/vterrors.Errorf([^,], /fmt.Errorf(/g' .go dependency/sqltypes/*.go
	131	sed -i '.bak' 's/vterrors.New([^,], /errors.New(/g' .go dependency/sqltypes/*.go
	132	```
	133
	134	### Testing
	135
	136	```bash
	137	VITESS=${GOPATH?}/src/vitess.io/vitess/go/
	138	XWB1989=${GOPATH?}/src/github.com/xwb1989/sqlparser/
	139
	140	cd ${XWB1989?}
	141
	142	# Test, fix and repeat
	143	go test ./...
	144
	145	# Finally make some diffs (for later reference)
	146	diff -u ${VITESS?}/sqltypes/ ${XWB1989?}/dependency/sqltypes/ > ${XWB1989?}/patches/sqltypes.patch
	147	diff -u ${VITESS?}/bytes2/ ${XWB1989?}/dependency/bytes2/ > ${XWB1989?}/patches/bytes2.patch
	148	diff -u ${VITESS?}/vt/proto/query/ ${XWB1989?}/dependency/querypb/ > ${XWB1989?}/patches/querypb.patch
	149	diff -u ${VITESS?}/vt/sqlparser/ ${XWB1989?}/ > ${XWB1989?}/patches/sqlparser.patch
	150	```

vendor/github.com/xwb1989/sqlparser/analyzer.go+343−0

@@ -0,0 +1,343 @@
	1	/*
	2	Copyright 2017 Google Inc.
	3
	4	Licensed under the Apache License, Version 2.0 (the "License");
	5	you may not use this file except in compliance with the License.
	6	You may obtain a copy of the License at
	7
	8	http://www.apache.org/licenses/LICENSE-2.0
	9
	10	Unless required by applicable law or agreed to in writing, software
	11	distributed under the License is distributed on an "AS IS" BASIS,
	12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	See the License for the specific language governing permissions and
	14	limitations under the License.
	15	*/
	16
	17	package sqlparser
	18
	19	// analyzer.go contains utility analysis functions.
	20
	21	import (
	22	"errors"
	23	"fmt"
	24	"strconv"
	25	"strings"
	26	"unicode"
	27
	28	"github.com/xwb1989/sqlparser/dependency/sqltypes"
	29	)
	30
	31	// These constants are used to identify the SQL statement type.
	32	const (
	33	StmtSelect = iota
	34	StmtStream
	35	StmtInsert
	36	StmtReplace
	37	StmtUpdate
	38	StmtDelete
	39	StmtDDL
	40	StmtBegin
	41	StmtCommit
	42	StmtRollback
	43	StmtSet
	44	StmtShow
	45	StmtUse
	46	StmtOther
	47	StmtUnknown
	48	StmtComment
	49	)
	50
	51	// Preview analyzes the beginning of the query using a simpler and faster
	52	// textual comparison to identify the statement type.
	53	func Preview(sql string) int {
	54	trimmed := StripLeadingComments(sql)
	55
	56	firstWord := trimmed
	57	if end := strings.IndexFunc(trimmed, unicode.IsSpace); end != -1 {
	58	firstWord = trimmed[:end]
	59	}
	60	firstWord = strings.TrimLeftFunc(firstWord, func(r rune) bool { return !unicode.IsLetter(r) })
	61	// Comparison is done in order of priority.
	62	loweredFirstWord := strings.ToLower(firstWord)
	63	switch loweredFirstWord {
	64	case "select":
	65	return StmtSelect
	66	case "stream":
	67	return StmtStream
	68	case "insert":
	69	return StmtInsert
	70	case "replace":
	71	return StmtReplace
	72	case "update":
	73	return StmtUpdate
	74	case "delete":
	75	return StmtDelete
	76	}
	77	// For the following statements it is not sufficient to rely
	78	// on loweredFirstWord. This is because they are not statements
	79	// in the grammar and we are relying on Preview to parse them.
	80	// For instance, we don't want: "BEGIN JUNK" to be parsed
	81	// as StmtBegin.
	82	trimmedNoComments, _ := SplitMarginComments(trimmed)
	83	switch strings.ToLower(trimmedNoComments) {
	84	case "begin", "start transaction":
	85	return StmtBegin
	86	case "commit":
	87	return StmtCommit
	88	case "rollback":
	89	return StmtRollback
	90	}
	91	switch loweredFirstWord {
	92	case "create", "alter", "rename", "drop", "truncate":
	93	return StmtDDL
	94	case "set":
	95	return StmtSet
	96	case "show":
	97	return StmtShow
	98	case "use":
	99	return StmtUse
	100	case "analyze", "describe", "desc", "explain", "repair", "optimize":
	101	return StmtOther
	102	}
	103	if strings.Index(trimmed, "/*!") == 0 {
	104	return StmtComment
	105	}
	106	return StmtUnknown
	107	}
	108
	109	// StmtType returns the statement type as a string
	110	func StmtType(stmtType int) string {
	111	switch stmtType {
	112	case StmtSelect:
	113	return "SELECT"
	114	case StmtStream:
	115	return "STREAM"
	116	case StmtInsert:
	117	return "INSERT"
	118	case StmtReplace:
	119	return "REPLACE"
	120	case StmtUpdate:
	121	return "UPDATE"
	122	case StmtDelete:
	123	return "DELETE"
	124	case StmtDDL:
	125	return "DDL"
	126	case StmtBegin:
	127	return "BEGIN"
	128	case StmtCommit:
	129	return "COMMIT"
	130	case StmtRollback:
	131	return "ROLLBACK"
	132	case StmtSet:
	133	return "SET"
	134	case StmtShow:
	135	return "SHOW"
	136	case StmtUse:
	137	return "USE"
	138	case StmtOther:
	139	return "OTHER"
	140	default:
	141	return "UNKNOWN"
	142	}
	143	}
	144
	145	// IsDML returns true if the query is an INSERT, UPDATE or DELETE statement.
	146	func IsDML(sql string) bool {
	147	switch Preview(sql) {
	148	case StmtInsert, StmtReplace, StmtUpdate, StmtDelete:
	149	return true
	150	}
	151	return false
	152	}
	153
	154	// GetTableName returns the table name from the SimpleTableExpr
	155	// only if it's a simple expression. Otherwise, it returns "".
	156	func GetTableName(node SimpleTableExpr) TableIdent {
	157	if n, ok := node.(TableName); ok && n.Qualifier.IsEmpty() {
	158	return n.Name
	159	}
	160	// sub-select or '.' expression
	161	return NewTableIdent("")
	162	}
	163
	164	// IsColName returns true if the Expr is a *ColName.
	165	func IsColName(node Expr) bool {
	166	_, ok := node.(*ColName)
	167	return ok
	168	}
	169
	170	// IsValue returns true if the Expr is a string, integral or value arg.
	171	// NULL is not considered to be a value.
	172	func IsValue(node Expr) bool {
	173	switch v := node.(type) {
	174	case *SQLVal:
	175	switch v.Type {
	176	case StrVal, HexVal, IntVal, ValArg:
	177	return true
	178	}
	179	}
	180	return false
	181	}
	182
	183	// IsNull returns true if the Expr is SQL NULL
	184	func IsNull(node Expr) bool {
	185	switch node.(type) {
	186	case *NullVal:
	187	return true
	188	}
	189	return false
	190	}
	191
	192	// IsSimpleTuple returns true if the Expr is a ValTuple that
	193	// contains simple values or if it's a list arg.
	194	func IsSimpleTuple(node Expr) bool {
	195	switch vals := node.(type) {
	196	case ValTuple:
	197	for _, n := range vals {
	198	if !IsValue(n) {
	199	return false
	200	}
	201	}
	202	return true
	203	case ListArg:
	204	return true
	205	}
	206	// It's a subquery
	207	return false
	208	}
	209
	210	// NewPlanValue builds a sqltypes.PlanValue from an Expr.
	211	func NewPlanValue(node Expr) (sqltypes.PlanValue, error) {
	212	switch node := node.(type) {
	213	case *SQLVal:
	214	switch node.Type {
	215	case ValArg:
	216	return sqltypes.PlanValue{Key: string(node.Val[1:])}, nil
	217	case IntVal:
	218	n, err := sqltypes.NewIntegral(string(node.Val))
	219	if err != nil {
	220	return sqltypes.PlanValue{}, fmt.Errorf("%v", err)
	221	}
	222	return sqltypes.PlanValue{Value: n}, nil
	223	case StrVal:
	224	return sqltypes.PlanValue{Value: sqltypes.MakeTrusted(sqltypes.VarBinary, node.Val)}, nil
	225	case HexVal:
	226	v, err := node.HexDecode()
	227	if err != nil {
	228	return sqltypes.PlanValue{}, fmt.Errorf("%v", err)
	229	}
	230	return sqltypes.PlanValue{Value: sqltypes.MakeTrusted(sqltypes.VarBinary, v)}, nil
	231	}
	232	case ListArg:
	233	return sqltypes.PlanValue{ListKey: string(node[2:])}, nil
	234	case ValTuple:
	235	pv := sqltypes.PlanValue{
	236	Values: make([]sqltypes.PlanValue, 0, len(node)),
	237	}
	238	for _, val := range node {
	239	innerpv, err := NewPlanValue(val)
	240	if err != nil {
	241	return sqltypes.PlanValue{}, err
	242	}
	243	if innerpv.ListKey != "" \|\| innerpv.Values != nil {
	244	return sqltypes.PlanValue{}, errors.New("unsupported: nested lists")
	245	}
	246	pv.Values = append(pv.Values, innerpv)
	247	}
	248	return pv, nil
	249	case *NullVal:
	250	return sqltypes.PlanValue{}, nil
	251	}
	252	return sqltypes.PlanValue{}, fmt.Errorf("expression is too complex '%v'", String(node))
	253	}
	254
	255	// StringIn is a convenience function that returns
	256	// true if str matches any of the values.
	257	func StringIn(str string, values ...string) bool {
	258	for _, val := range values {
	259	if str == val {
	260	return true
	261	}
	262	}
	263	return false
	264	}
	265
	266	// SetKey is the extracted key from one SetExpr
	267	type SetKey struct {
	268	Key string
	269	Scope string
	270	}
	271
	272	// ExtractSetValues returns a map of key-value pairs
	273	// if the query is a SET statement. Values can be bool, int64 or string.
	274	// Since set variable names are case insensitive, all keys are returned
	275	// as lower case.
	276	func ExtractSetValues(sql string) (keyValues map[SetKey]interface{}, scope string, err error) {
	277	stmt, err := Parse(sql)
	278	if err != nil {
	279	return nil, "", err
	280	}
	281	setStmt, ok := stmt.(*Set)
	282	if !ok {
	283	return nil, "", fmt.Errorf("ast did not yield *sqlparser.Set: %T", stmt)
	284	}
	285	result := make(map[SetKey]interface{})
	286	for _, expr := range setStmt.Exprs {
	287	scope := SessionStr
	288	key := expr.Name.Lowered()
	289	switch {
	290	case strings.HasPrefix(key, "@@global."):
	291	scope = GlobalStr
	292	key = strings.TrimPrefix(key, "@@global.")
	293	case strings.HasPrefix(key, "@@session."):
	294	key = strings.TrimPrefix(key, "@@session.")
	295	case strings.HasPrefix(key, "@@"):
	296	key = strings.TrimPrefix(key, "@@")
	297	}
	298
	299	if strings.HasPrefix(expr.Name.Lowered(), "@@") {
	300	if setStmt.Scope != "" && scope != "" {
	301	return nil, "", fmt.Errorf("unsupported in set: mixed using of variable scope")
	302	}
	303	_, out := NewStringTokenizer(key).Scan()
	304	key = string(out)
	305	}
	306
	307	setKey := SetKey{
	308	Key: key,
	309	Scope: scope,
	310	}
	311
	312	switch expr := expr.Expr.(type) {
	313	case *SQLVal:
	314	switch expr.Type {
	315	case StrVal:
	316	result[setKey] = strings.ToLower(string(expr.Val))
	317	case IntVal:
	318	num, err := strconv.ParseInt(string(expr.Val), 0, 64)
	319	if err != nil {
	320	return nil, "", err
	321	}
	322	result[setKey] = num
	323	default:
	324	return nil, "", fmt.Errorf("invalid value type: %v", String(expr))
	325	}
	326	case BoolVal:
	327	var val int64
	328	if expr {
	329	val = 1
	330	}
	331	result[setKey] = val
	332	case *ColName:
	333	result[setKey] = expr.Name.String()
	334	case *NullVal:
	335	result[setKey] = nil
	336	case *Default:
	337	result[setKey] = "default"
	338	default:
	339	return nil, "", fmt.Errorf("invalid syntax: %s", String(expr))
	340	}
	341	}
	342	return result, strings.ToLower(setStmt.Scope), nil
	343	}

vendor/github.com/xwb1989/sqlparser/ast.go+3450−0

@@ -0,0 +1,3450 @@
	1	/*
	2	Copyright 2017 Google Inc.
	3
	4	Licensed under the Apache License, Version 2.0 (the "License");
	5	you may not use this file except in compliance with the License.
	6	You may obtain a copy of the License at
	7
	8	http://www.apache.org/licenses/LICENSE-2.0
	9
	10	Unless required by applicable law or agreed to in writing, software
	11	distributed under the License is distributed on an "AS IS" BASIS,
	12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	See the License for the specific language governing permissions and
	14	limitations under the License.
	15	*/
	16
	17	package sqlparser
	18
	19	import (
	20	"bytes"
	21	"encoding/hex"
	22	"encoding/json"
	23	"fmt"
	24	"io"
	25	"log"
	26	"strings"
	27
	28	"github.com/xwb1989/sqlparser/dependency/querypb"
	29	"github.com/xwb1989/sqlparser/dependency/sqltypes"
	30	)
	31
	32	// Instructions for creating new types: If a type
	33	// needs to satisfy an interface, declare that function
	34	// along with that interface. This will help users
	35	// identify the list of types to which they can assert
	36	// those interfaces.
	37	// If the member of a type has a string with a predefined
	38	// list of values, declare those values as const following
	39	// the type.
	40	// For interfaces that define dummy functions to consolidate
	41	// a set of types, define the function as iTypeName.
	42	// This will help avoid name collisions.
	43
	44	// Parse parses the SQL in full and returns a Statement, which
	45	// is the AST representation of the query. If a DDL statement
	46	// is partially parsed but still contains a syntax error, the
	47	// error is ignored and the DDL is returned anyway.
	48	func Parse(sql string) (Statement, error) {
	49	tokenizer := NewStringTokenizer(sql)
	50	if yyParse(tokenizer) != 0 {
	51	if tokenizer.partialDDL != nil {
	52	log.Printf("ignoring error parsing DDL '%s': %v", sql, tokenizer.LastError)
	53	tokenizer.ParseTree = tokenizer.partialDDL
	54	return tokenizer.ParseTree, nil
	55	}
	56	return nil, tokenizer.LastError
	57	}
	58	return tokenizer.ParseTree, nil
	59	}
	60
	61	// ParseStrictDDL is the same as Parse except it errors on
	62	// partially parsed DDL statements.
	63	func ParseStrictDDL(sql string) (Statement, error) {
	64	tokenizer := NewStringTokenizer(sql)
	65	if yyParse(tokenizer) != 0 {
	66	return nil, tokenizer.LastError
	67	}
	68	return tokenizer.ParseTree, nil
	69	}
	70
	71	// ParseNext parses a single SQL statement from the tokenizer
	72	// returning a Statement which is the AST representation of the query.
	73	// The tokenizer will always read up to the end of the statement, allowing for
	74	// the next call to ParseNext to parse any subsequent SQL statements. When
	75	// there are no more statements to parse, a error of io.EOF is returned.
	76	func ParseNext(tokenizer *Tokenizer) (Statement, error) {
	77	if tokenizer.lastChar == ';' {
	78	tokenizer.next()
	79	tokenizer.skipBlank()
	80	}
	81	if tokenizer.lastChar == eofChar {
	82	return nil, io.EOF
	83	}
	84
	85	tokenizer.reset()
	86	tokenizer.multi = true
	87	if yyParse(tokenizer) != 0 {
	88	if tokenizer.partialDDL != nil {
	89	tokenizer.ParseTree = tokenizer.partialDDL
	90	return tokenizer.ParseTree, nil
	91	}
	92	return nil, tokenizer.LastError
	93	}
	94	return tokenizer.ParseTree, nil
	95	}
	96
	97	// SplitStatement returns the first sql statement up to either a ; or EOF
	98	// and the remainder from the given buffer
	99	func SplitStatement(blob string) (string, string, error) {
	100	tokenizer := NewStringTokenizer(blob)
	101	tkn := 0
	102	for {
	103	tkn, _ = tokenizer.Scan()
	104	if tkn == 0 \|\| tkn == ';' \|\| tkn == eofChar {
	105	break
	106	}
	107	}
	108	if tokenizer.LastError != nil {
	109	return "", "", tokenizer.LastError
	110	}
	111	if tkn == ';' {
	112	return blob[:tokenizer.Position-2], blob[tokenizer.Position-1:], nil
	113	}
	114	return blob, "", nil
	115	}
	116
	117	// SplitStatementToPieces split raw sql statement that may have multi sql pieces to sql pieces
	118	// returns the sql pieces blob contains; or error if sql cannot be parsed
	119	func SplitStatementToPieces(blob string) (pieces []string, err error) {
	120	pieces = make([]string, 0, 16)
	121	tokenizer := NewStringTokenizer(blob)
	122
	123	tkn := 0
	124	var stmt string
	125	stmtBegin := 0
	126	for {
	127	tkn, _ = tokenizer.Scan()
	128	if tkn == ';' {
	129	stmt = blob[stmtBegin : tokenizer.Position-2]
	130	pieces = append(pieces, stmt)
	131	stmtBegin = tokenizer.Position - 1
	132
	133	} else if tkn == 0 \|\| tkn == eofChar {
	134	blobTail := tokenizer.Position - 2
	135
	136	if stmtBegin < blobTail {
	137	stmt = blob[stmtBegin : blobTail+1]
	138	pieces = append(pieces, stmt)
	139	}
	140	break
	141	}
	142	}
	143
	144	err = tokenizer.LastError
	145	return
	146	}
	147
	148	// SQLNode defines the interface for all nodes
	149	// generated by the parser.
	150	type SQLNode interface {
	151	Format(buf *TrackedBuffer)
	152	// walkSubtree calls visit on all underlying nodes
	153	// of the subtree, but not the current one. Walking
	154	// must be interrupted if visit returns an error.
	155	walkSubtree(visit Visit) error
	156	}
	157
	158	// Visit defines the signature of a function that
	159	// can be used to visit all nodes of a parse tree.
	160	type Visit func(node SQLNode) (kontinue bool, err error)
	161
	162	// Walk calls visit on every node.
	163	// If visit returns true, the underlying nodes
	164	// are also visited. If it returns an error, walking
	165	// is interrupted, and the error is returned.
	166	func Walk(visit Visit, nodes ...SQLNode) error {
	167	for _, node := range nodes {
	168	if node == nil {
	169	continue
	170	}
	171	kontinue, err := visit(node)
	172	if err != nil {
	173	return err
	174	}
	175	if kontinue {
	176	err = node.walkSubtree(visit)
	177	if err != nil {
	178	return err
	179	}
	180	}
	181	}
	182	return nil
	183	}
	184
	185	// String returns a string representation of an SQLNode.
	186	func String(node SQLNode) string {
	187	if node == nil {
	188	return "<nil>"
	189	}
	190
	191	buf := NewTrackedBuffer(nil)
	192	buf.Myprintf("%v", node)
	193	return buf.String()
	194	}
	195
	196	// Append appends the SQLNode to the buffer.
	197	func Append(buf *bytes.Buffer, node SQLNode) {
	198	tbuf := &TrackedBuffer{
	199	Buffer: buf,
	200	}
	201	node.Format(tbuf)
	202	}
	203
	204	// Statement represents a statement.
	205	type Statement interface {
	206	iStatement()
	207	SQLNode
	208	}
	209
	210	func (*Union) iStatement() {}
	211	func (*Select) iStatement() {}
	212	func (*Stream) iStatement() {}
	213	func (*Insert) iStatement() {}
	214	func (*Update) iStatement() {}
	215	func (*Delete) iStatement() {}
	216	func (*Set) iStatement() {}
	217	func (*DBDDL) iStatement() {}
	218	func (*DDL) iStatement() {}
	219	func (*Show) iStatement() {}
	220	func (*Use) iStatement() {}
	221	func (*Begin) iStatement() {}
	222	func (*Commit) iStatement() {}
	223	func (*Rollback) iStatement() {}
	224	func (*OtherRead) iStatement() {}
	225	func (*OtherAdmin) iStatement() {}
	226
	227	// ParenSelect can actually not be a top level statement,
	228	// but we have to allow it because it's a requirement
	229	// of SelectStatement.
	230	func (*ParenSelect) iStatement() {}
	231
	232	// SelectStatement any SELECT statement.
	233	type SelectStatement interface {
	234	iSelectStatement()
	235	iStatement()
	236	iInsertRows()
	237	AddOrder(*Order)
	238	SetLimit(*Limit)
	239	SQLNode
	240	}
	241
	242	func (*Select) iSelectStatement() {}
	243	func (*Union) iSelectStatement() {}
	244	func (*ParenSelect) iSelectStatement() {}
	245
	246	// Select represents a SELECT statement.
	247	type Select struct {
	248	Cache string
	249	Comments Comments
	250	Distinct string
	251	Hints string
	252	SelectExprs SelectExprs
	253	From TableExprs
	254	Where *Where
	255	GroupBy GroupBy
	256	Having *Where
	257	OrderBy OrderBy
	258	Limit *Limit
	259	Lock string
	260	}
	261
	262	// Select.Distinct
	263	const (
	264	DistinctStr = "distinct "
	265	StraightJoinHint = "straight_join "
	266	)
	267
	268	// Select.Lock
	269	const (
	270	ForUpdateStr = " for update"
	271	ShareModeStr = " lock in share mode"
	272	)
	273
	274	// Select.Cache
	275	const (
	276	SQLCacheStr = "sql_cache "
	277	SQLNoCacheStr = "sql_no_cache "
	278	)
	279
	280	// AddOrder adds an order by element
	281	func (node Select) AddOrder(order Order) {
	282	node.OrderBy = append(node.OrderBy, order)
	283	}
	284
	285	// SetLimit sets the limit clause
	286	func (node Select) SetLimit(limit Limit) {
	287	node.Limit = limit
	288	}
	289
	290	// Format formats the node.
	291	func (node Select) Format(buf TrackedBuffer) {
	292	buf.Myprintf("select %v%s%s%s%v from %v%v%v%v%v%v%s",
	293	node.Comments, node.Cache, node.Distinct, node.Hints, node.SelectExprs,
	294	node.From, node.Where,
	295	node.GroupBy, node.Having, node.OrderBy,
	296	node.Limit, node.Lock)
	297	}
	298
	299	func (node *Select) walkSubtree(visit Visit) error {
	300	if node == nil {
	301	return nil
	302	}
	303	return Walk(
	304	visit,
	305	node.Comments,
	306	node.SelectExprs,
	307	node.From,
	308	node.Where,
	309	node.GroupBy,
	310	node.Having,
	311	node.OrderBy,
	312	node.Limit,
	313	)
	314	}
	315
	316	// AddWhere adds the boolean expression to the
	317	// WHERE clause as an AND condition. If the expression
	318	// is an OR clause, it parenthesizes it. Currently,
	319	// the OR operator is the only one that's lower precedence
	320	// than AND.
	321	func (node *Select) AddWhere(expr Expr) {
	322	if _, ok := expr.(*OrExpr); ok {
	323	expr = &ParenExpr{Expr: expr}
	324	}
	325	if node.Where == nil {
	326	node.Where = &Where{
	327	Type: WhereStr,
	328	Expr: expr,
	329	}
	330	return
	331	}
	332	node.Where.Expr = &AndExpr{
	333	Left: node.Where.Expr,
	334	Right: expr,
	335	}
	336	return
	337	}
	338
	339	// AddHaving adds the boolean expression to the
	340	// HAVING clause as an AND condition. If the expression
	341	// is an OR clause, it parenthesizes it. Currently,
	342	// the OR operator is the only one that's lower precedence
	343	// than AND.
	344	func (node *Select) AddHaving(expr Expr) {
	345	if _, ok := expr.(*OrExpr); ok {
	346	expr = &ParenExpr{Expr: expr}
	347	}
	348	if node.Having == nil {
	349	node.Having = &Where{
	350	Type: HavingStr,
	351	Expr: expr,
	352	}
	353	return
	354	}
	355	node.Having.Expr = &AndExpr{
	356	Left: node.Having.Expr,
	357	Right: expr,
	358	}
	359	return
	360	}
	361
	362	// ParenSelect is a parenthesized SELECT statement.
	363	type ParenSelect struct {
	364	Select SelectStatement
	365	}
	366
	367	// AddOrder adds an order by element
	368	func (node ParenSelect) AddOrder(order Order) {
	369	panic("unreachable")
	370	}
	371
	372	// SetLimit sets the limit clause
	373	func (node ParenSelect) SetLimit(limit Limit) {
	374	panic("unreachable")
	375	}
	376
	377	// Format formats the node.
	378	func (node ParenSelect) Format(buf TrackedBuffer) {
	379	buf.Myprintf("(%v)", node.Select)
	380	}
	381
	382	func (node *ParenSelect) walkSubtree(visit Visit) error {
	383	if node == nil {
	384	return nil
	385	}
	386	return Walk(
	387	visit,
	388	node.Select,
	389	)
	390	}
	391
	392	// Union represents a UNION statement.
	393	type Union struct {
	394	Type string
	395	Left, Right SelectStatement
	396	OrderBy OrderBy
	397	Limit *Limit
	398	Lock string
	399	}
	400
	401	// Union.Type
	402	const (
	403	UnionStr = "union"
	404	UnionAllStr = "union all"
	405	UnionDistinctStr = "union distinct"
	406	)
	407
	408	// AddOrder adds an order by element
	409	func (node Union) AddOrder(order Order) {
	410	node.OrderBy = append(node.OrderBy, order)
	411	}
	412
	413	// SetLimit sets the limit clause
	414	func (node Union) SetLimit(limit Limit) {
	415	node.Limit = limit
	416	}
	417
	418	// Format formats the node.
	419	func (node Union) Format(buf TrackedBuffer) {
	420	buf.Myprintf("%v %s %v%v%v%s", node.Left, node.Type, node.Right,
	421	node.OrderBy, node.Limit, node.Lock)
	422	}
	423
	424	func (node *Union) walkSubtree(visit Visit) error {
	425	if node == nil {
	426	return nil
	427	}
	428	return Walk(
	429	visit,
	430	node.Left,
	431	node.Right,
	432	)
	433	}
	434
	435	// Stream represents a SELECT statement.
	436	type Stream struct {
	437	Comments Comments
	438	SelectExpr SelectExpr
	439	Table TableName
	440	}
	441
	442	// Format formats the node.
	443	func (node Stream) Format(buf TrackedBuffer) {
	444	buf.Myprintf("stream %v%v from %v",
	445	node.Comments, node.SelectExpr, node.Table)
	446	}
	447
	448	func (node *Stream) walkSubtree(visit Visit) error {
	449	if node == nil {
	450	return nil
	451	}
	452	return Walk(
	453	visit,
	454	node.Comments,
	455	node.SelectExpr,
	456	node.Table,
	457	)
	458	}
	459
	460	// Insert represents an INSERT or REPLACE statement.
	461	// Per the MySQL docs, http://dev.mysql.com/doc/refman/5.7/en/replace.html
	462	// Replace is the counterpart to `INSERT IGNORE`, and works exactly like a
	463	// normal INSERT except if the row exists. In that case it first deletes
	464	// the row and re-inserts with new values. For that reason we keep it as an Insert struct.
	465	// Replaces are currently disallowed in sharded schemas because
	466	// of the implications the deletion part may have on vindexes.
	467	// If you add fields here, consider adding them to calls to validateSubquerySamePlan.
	468	type Insert struct {
	469	Action string
	470	Comments Comments
	471	Ignore string
	472	Table TableName
	473	Partitions Partitions
	474	Columns Columns
	475	Rows InsertRows
	476	OnDup OnDup
	477	}
	478
	479	// DDL strings.
	480	const (
	481	InsertStr = "insert"
	482	ReplaceStr = "replace"
	483	)
	484
	485	// Format formats the node.
	486	func (node Insert) Format(buf TrackedBuffer) {
	487	buf.Myprintf("%s %v%sinto %v%v%v %v%v",
	488	node.Action,
	489	node.Comments, node.Ignore,
	490	node.Table, node.Partitions, node.Columns, node.Rows, node.OnDup)
	491	}
	492
	493	func (node *Insert) walkSubtree(visit Visit) error {
	494	if node == nil {
	495	return nil
	496	}
	497	return Walk(
	498	visit,
	499	node.Comments,
	500	node.Table,
	501	node.Columns,
	502	node.Rows,
	503	node.OnDup,
	504	)
	505	}
	506
	507	// InsertRows represents the rows for an INSERT statement.
	508	type InsertRows interface {
	509	iInsertRows()
	510	SQLNode
	511	}
	512
	513	func (*Select) iInsertRows() {}
	514	func (*Union) iInsertRows() {}
	515	func (Values) iInsertRows() {}
	516	func (*ParenSelect) iInsertRows() {}
	517
	518	// Update represents an UPDATE statement.
	519	// If you add fields here, consider adding them to calls to validateSubquerySamePlan.
	520	type Update struct {
	521	Comments Comments
	522	TableExprs TableExprs
	523	Exprs UpdateExprs
	524	Where *Where
	525	OrderBy OrderBy
	526	Limit *Limit
	527	}
	528
	529	// Format formats the node.
	530	func (node Update) Format(buf TrackedBuffer) {
	531	buf.Myprintf("update %v%v set %v%v%v%v",
	532	node.Comments, node.TableExprs,
	533	node.Exprs, node.Where, node.OrderBy, node.Limit)
	534	}
	535
	536	func (node *Update) walkSubtree(visit Visit) error {
	537	if node == nil {
	538	return nil
	539	}
	540	return Walk(
	541	visit,
	542	node.Comments,
	543	node.TableExprs,
	544	node.Exprs,
	545	node.Where,
	546	node.OrderBy,
	547	node.Limit,
	548	)
	549	}
	550
	551	// Delete represents a DELETE statement.
	552	// If you add fields here, consider adding them to calls to validateSubquerySamePlan.
	553	type Delete struct {
	554	Comments Comments
	555	Targets TableNames
	556	TableExprs TableExprs
	557	Partitions Partitions
	558	Where *Where
	559	OrderBy OrderBy
	560	Limit *Limit
	561	}
	562
	563	// Format formats the node.
	564	func (node Delete) Format(buf TrackedBuffer) {
	565	buf.Myprintf("delete %v", node.Comments)
	566	if node.Targets != nil {
	567	buf.Myprintf("%v ", node.Targets)
	568	}
	569	buf.Myprintf("from %v%v%v%v%v", node.TableExprs, node.Partitions, node.Where, node.OrderBy, node.Limit)
	570	}
	571
	572	func (node *Delete) walkSubtree(visit Visit) error {
	573	if node == nil {
	574	return nil
	575	}
	576	return Walk(
	577	visit,
	578	node.Comments,
	579	node.Targets,
	580	node.TableExprs,
	581	node.Where,
	582	node.OrderBy,
	583	node.Limit,
	584	)
	585	}
	586
	587	// Set represents a SET statement.
	588	type Set struct {
	589	Comments Comments
	590	Exprs SetExprs
	591	Scope string
	592	}
	593
	594	// Set.Scope or Show.Scope
	595	const (
	596	SessionStr = "session"
	597	GlobalStr = "global"
	598	)
	599
	600	// Format formats the node.
	601	func (node Set) Format(buf TrackedBuffer) {
	602	if node.Scope == "" {
	603	buf.Myprintf("set %v%v", node.Comments, node.Exprs)
	604	} else {
	605	buf.Myprintf("set %v%s %v", node.Comments, node.Scope, node.Exprs)
	606	}
	607	}
	608
	609	func (node *Set) walkSubtree(visit Visit) error {
	610	if node == nil {
	611	return nil
	612	}
	613	return Walk(
	614	visit,
	615	node.Comments,
	616	node.Exprs,
	617	)
	618	}
	619
	620	// DBDDL represents a CREATE, DROP database statement.
	621	type DBDDL struct {
	622	Action string
	623	DBName string
	624	IfExists bool
	625	Collate string
	626	Charset string
	627	}
	628
	629	// Format formats the node.
	630	func (node DBDDL) Format(buf TrackedBuffer) {
	631	switch node.Action {
	632	case CreateStr:
	633	buf.WriteString(fmt.Sprintf("%s database %s", node.Action, node.DBName))
	634	case DropStr:
	635	exists := ""
	636	if node.IfExists {
	637	exists = " if exists"
	638	}
	639	buf.WriteString(fmt.Sprintf("%s database%s %v", node.Action, exists, node.DBName))
	640	}
	641	}
	642
	643	// walkSubtree walks the nodes of the subtree.
	644	func (node *DBDDL) walkSubtree(visit Visit) error {
	645	return nil
	646	}
	647
	648	// DDL represents a CREATE, ALTER, DROP, RENAME or TRUNCATE statement.
	649	// Table is set for AlterStr, DropStr, RenameStr, TruncateStr
	650	// NewName is set for AlterStr, CreateStr, RenameStr.
	651	// VindexSpec is set for CreateVindexStr, DropVindexStr, AddColVindexStr, DropColVindexStr
	652	// VindexCols is set for AddColVindexStr
	653	type DDL struct {
	654	Action string
	655	Table TableName
	656	NewName TableName
	657	IfExists bool
	658	TableSpec *TableSpec
	659	PartitionSpec *PartitionSpec
	660	VindexSpec *VindexSpec
	661	VindexCols []ColIdent
	662	}
	663
	664	// DDL strings.
	665	const (
	666	CreateStr = "create"
	667	AlterStr = "alter"
	668	DropStr = "drop"
	669	RenameStr = "rename"
	670	TruncateStr = "truncate"
	671	CreateVindexStr = "create vindex"
	672	AddColVindexStr = "add vindex"
	673	DropColVindexStr = "drop vindex"
	674
	675	// Vindex DDL param to specify the owner of a vindex
	676	VindexOwnerStr = "owner"
	677	)
	678
	679	// Format formats the node.
	680	func (node DDL) Format(buf TrackedBuffer) {
	681	switch node.Action {
	682	case CreateStr:
	683	if node.TableSpec == nil {
	684	buf.Myprintf("%s table %v", node.Action, node.NewName)
	685	} else {
	686	buf.Myprintf("%s table %v %v", node.Action, node.NewName, node.TableSpec)
	687	}
	688	case DropStr:
	689	exists := ""
	690	if node.IfExists {
	691	exists = " if exists"
	692	}
	693	buf.Myprintf("%s table%s %v", node.Action, exists, node.Table)
	694	case RenameStr:
	695	buf.Myprintf("%s table %v to %v", node.Action, node.Table, node.NewName)
	696	case AlterStr:
	697	if node.PartitionSpec != nil {
	698	buf.Myprintf("%s table %v %v", node.Action, node.Table, node.PartitionSpec)
	699	} else {
	700	buf.Myprintf("%s table %v", node.Action, node.Table)
	701	}
	702	case CreateVindexStr:
	703	buf.Myprintf("%s %v %v", node.Action, node.VindexSpec.Name, node.VindexSpec)
	704	case AddColVindexStr:
	705	buf.Myprintf("alter table %v %s %v (", node.Table, node.Action, node.VindexSpec.Name)
	706	for i, col := range node.VindexCols {
	707	if i != 0 {
	708	buf.Myprintf(", %v", col)
	709	} else {
	710	buf.Myprintf("%v", col)
	711	}
	712	}
	713	buf.Myprintf(")")
	714	if node.VindexSpec.Type.String() != "" {
	715	buf.Myprintf(" %v", node.VindexSpec)
	716	}
	717	case DropColVindexStr:
	718	buf.Myprintf("alter table %v %s %v", node.Table, node.Action, node.VindexSpec.Name)
	719	default:
	720	buf.Myprintf("%s table %v", node.Action, node.Table)
	721	}
	722	}
	723
	724	func (node *DDL) walkSubtree(visit Visit) error {
	725	if node == nil {
	726	return nil
	727	}
	728	return Walk(
	729	visit,
	730	node.Table,
	731	node.NewName,
	732	)
	733	}
	734
	735	// Partition strings
	736	const (
	737	ReorganizeStr = "reorganize partition"
	738	)
	739
	740	// PartitionSpec describe partition actions (for alter and create)
	741	type PartitionSpec struct {
	742	Action string
	743	Name ColIdent
	744	Definitions []*PartitionDefinition
	745	}
	746
	747	// Format formats the node.
	748	func (node PartitionSpec) Format(buf TrackedBuffer) {
	749	switch node.Action {
	750	case ReorganizeStr:
	751	buf.Myprintf("%s %v into (", node.Action, node.Name)
	752	var prefix string
	753	for _, pd := range node.Definitions {
	754	buf.Myprintf("%s%v", prefix, pd)
	755	prefix = ", "
	756	}
	757	buf.Myprintf(")")
	758	default:
	759	panic("unimplemented")
	760	}
	761	}
	762
	763	func (node *PartitionSpec) walkSubtree(visit Visit) error {
	764	if node == nil {
	765	return nil
	766	}
	767	if err := Walk(visit, node.Name); err != nil {
	768	return err
	769	}
	770	for _, def := range node.Definitions {
	771	if err := Walk(visit, def); err != nil {
	772	return err
	773	}
	774	}
	775	return nil
	776	}
	777
	778	// PartitionDefinition describes a very minimal partition definition
	779	type PartitionDefinition struct {
	780	Name ColIdent
	781	Limit Expr
	782	Maxvalue bool
	783	}
	784
	785	// Format formats the node
	786	func (node PartitionDefinition) Format(buf TrackedBuffer) {
	787	if !node.Maxvalue {
	788	buf.Myprintf("partition %v values less than (%v)", node.Name, node.Limit)
	789	} else {
	790	buf.Myprintf("partition %v values less than (maxvalue)", node.Name)
	791	}
	792	}
	793
	794	func (node *PartitionDefinition) walkSubtree(visit Visit) error {
	795	if node == nil {
	796	return nil
	797	}
	798	return Walk(
	799	visit,
	800	node.Name,
	801	node.Limit,
	802	)
	803	}
	804
	805	// TableSpec describes the structure of a table from a CREATE TABLE statement
	806	type TableSpec struct {
	807	Columns []*ColumnDefinition
	808	Indexes []*IndexDefinition
	809	Options string
	810	}
	811
	812	// Format formats the node.
	813	func (ts TableSpec) Format(buf TrackedBuffer) {
	814	buf.Myprintf("(\n")
	815	for i, col := range ts.Columns {
	816	if i == 0 {
	817	buf.Myprintf("\t%v", col)
	818	} else {
	819	buf.Myprintf(",\n\t%v", col)
	820	}
	821	}
	822	for _, idx := range ts.Indexes {
	823	buf.Myprintf(",\n\t%v", idx)
	824	}
	825
	826	buf.Myprintf("\n)%s", strings.Replace(ts.Options, ", ", ",\n ", -1))
	827	}
	828
	829	// AddColumn appends the given column to the list in the spec
	830	func (ts TableSpec) AddColumn(cd ColumnDefinition) {
	831	ts.Columns = append(ts.Columns, cd)
	832	}
	833
	834	// AddIndex appends the given index to the list in the spec
	835	func (ts TableSpec) AddIndex(id IndexDefinition) {
	836	ts.Indexes = append(ts.Indexes, id)
	837	}
	838
	839	func (ts *TableSpec) walkSubtree(visit Visit) error {
	840	if ts == nil {
	841	return nil
	842	}
	843
	844	for _, n := range ts.Columns {
	845	if err := Walk(visit, n); err != nil {
	846	return err
	847	}
	848	}
	849
	850	for _, n := range ts.Indexes {
	851	if err := Walk(visit, n); err != nil {
	852	return err
	853	}
	854	}
	855
	856	return nil
	857	}
	858
	859	// ColumnDefinition describes a column in a CREATE TABLE statement
	860	type ColumnDefinition struct {
	861	Name ColIdent
	862	Type ColumnType
	863	}
	864
	865	// Format formats the node.
	866	func (col ColumnDefinition) Format(buf TrackedBuffer) {
	867	buf.Myprintf("%v %v", col.Name, &col.Type)
	868	}
	869
	870	func (col *ColumnDefinition) walkSubtree(visit Visit) error {
	871	if col == nil {
	872	return nil
	873	}
	874	return Walk(
	875	visit,
	876	col.Name,
	877	&col.Type,
	878	)
	879	}
	880
	881	// ColumnType represents a sql type in a CREATE TABLE statement
	882	// All optional fields are nil if not specified
	883	type ColumnType struct {
	884	// The base type string
	885	Type string
	886
	887	// Generic field options.
	888	NotNull BoolVal
	889	Autoincrement BoolVal
	890	Default *SQLVal
	891	OnUpdate *SQLVal
	892	Comment *SQLVal
	893
	894	// Numeric field options
	895	Length *SQLVal
	896	Unsigned BoolVal
	897	Zerofill BoolVal
	898	Scale *SQLVal
	899
	900	// Text field options
	901	Charset string
	902	Collate string
	903
	904	// Enum values
	905	EnumValues []string
	906
	907	// Key specification
	908	KeyOpt ColumnKeyOption
	909	}
	910
	911	// Format returns a canonical string representation of the type and all relevant options
	912	func (ct ColumnType) Format(buf TrackedBuffer) {
	913	buf.Myprintf("%s", ct.Type)
	914
	915	if ct.Length != nil && ct.Scale != nil {
	916	buf.Myprintf("(%v,%v)", ct.Length, ct.Scale)
	917
	918	} else if ct.Length != nil {
	919	buf.Myprintf("(%v)", ct.Length)
	920	}
	921
	922	if ct.EnumValues != nil {
	923	buf.Myprintf("(%s)", strings.Join(ct.EnumValues, ", "))
	924	}
	925
	926	opts := make([]string, 0, 16)
	927	if ct.Unsigned {
	928	opts = append(opts, keywordStrings[UNSIGNED])
	929	}
	930	if ct.Zerofill {
	931	opts = append(opts, keywordStrings[ZEROFILL])
	932	}
	933	if ct.Charset != "" {
	934	opts = append(opts, keywordStrings[CHARACTER], keywordStrings[SET], ct.Charset)
	935	}
	936	if ct.Collate != "" {
	937	opts = append(opts, keywordStrings[COLLATE], ct.Collate)
	938	}
	939	if ct.NotNull {
	940	opts = append(opts, keywordStrings[NOT], keywordStrings[NULL])
	941	}
	942	if ct.Default != nil {
	943	opts = append(opts, keywordStrings[DEFAULT], String(ct.Default))
	944	}
	945	if ct.OnUpdate != nil {
	946	opts = append(opts, keywordStrings[ON], keywordStrings[UPDATE], String(ct.OnUpdate))
	947	}
	948	if ct.Autoincrement {
	949	opts = append(opts, keywordStrings[AUTO_INCREMENT])
	950	}
	951	if ct.Comment != nil {
	952	opts = append(opts, keywordStrings[COMMENT_KEYWORD], String(ct.Comment))
	953	}
	954	if ct.KeyOpt == colKeyPrimary {
	955	opts = append(opts, keywordStrings[PRIMARY], keywordStrings[KEY])
	956	}
	957	if ct.KeyOpt == colKeyUnique {
	958	opts = append(opts, keywordStrings[UNIQUE])
	959	}
	960	if ct.KeyOpt == colKeyUniqueKey {
	961	opts = append(opts, keywordStrings[UNIQUE], keywordStrings[KEY])
	962	}
	963	if ct.KeyOpt == colKeySpatialKey {
	964	opts = append(opts, keywordStrings[SPATIAL], keywordStrings[KEY])
	965	}
	966	if ct.KeyOpt == colKey {
	967	opts = append(opts, keywordStrings[KEY])
	968	}
	969
	970	if len(opts) != 0 {
	971	buf.Myprintf(" %s", strings.Join(opts, " "))
	972	}
	973	}
	974
	975	// DescribeType returns the abbreviated type information as required for
	976	// describe table
	977	func (ct *ColumnType) DescribeType() string {
	978	buf := NewTrackedBuffer(nil)
	979	buf.Myprintf("%s", ct.Type)
	980	if ct.Length != nil && ct.Scale != nil {
	981	buf.Myprintf("(%v,%v)", ct.Length, ct.Scale)
	982	} else if ct.Length != nil {
	983	buf.Myprintf("(%v)", ct.Length)
	984	}
	985
	986	opts := make([]string, 0, 16)
	987	if ct.Unsigned {
	988	opts = append(opts, keywordStrings[UNSIGNED])
	989	}
	990	if ct.Zerofill {
	991	opts = append(opts, keywordStrings[ZEROFILL])
	992	}
	993	if len(opts) != 0 {
	994	buf.Myprintf(" %s", strings.Join(opts, " "))
	995	}
	996	return buf.String()
	997	}
	998
	999	// SQLType returns the sqltypes type code for the given column
	1000	func (ct *ColumnType) SQLType() querypb.Type {
	1001	switch ct.Type {
	1002	case keywordStrings[TINYINT]:
	1003	if ct.Unsigned {
	1004	return sqltypes.Uint8
	1005	}
	1006	return sqltypes.Int8
	1007	case keywordStrings[SMALLINT]:
	1008	if ct.Unsigned {
	1009	return sqltypes.Uint16
	1010	}
	1011	return sqltypes.Int16
	1012	case keywordStrings[MEDIUMINT]:
	1013	if ct.Unsigned {
	1014	return sqltypes.Uint24
	1015	}
	1016	return sqltypes.Int24
	1017	case keywordStrings[INT]:
	1018	fallthrough
	1019	case keywordStrings[INTEGER]:
	1020	if ct.Unsigned {
	1021	return sqltypes.Uint32
	1022	}
	1023	return sqltypes.Int32
	1024	case keywordStrings[BIGINT]:
	1025	if ct.Unsigned {
	1026	return sqltypes.Uint64
	1027	}
	1028	return sqltypes.Int64
	1029	case keywordStrings[TEXT]:
	1030	return sqltypes.Text
	1031	case keywordStrings[TINYTEXT]:
	1032	return sqltypes.Text
	1033	case keywordStrings[MEDIUMTEXT]:
	1034	return sqltypes.Text
	1035	case keywordStrings[LONGTEXT]:
	1036	return sqltypes.Text
	1037	case keywordStrings[BLOB]:
	1038	return sqltypes.Blob
	1039	case keywordStrings[TINYBLOB]:
	1040	return sqltypes.Blob
	1041	case keywordStrings[MEDIUMBLOB]:
	1042	return sqltypes.Blob
	1043	case keywordStrings[LONGBLOB]:
	1044	return sqltypes.Blob
	1045	case keywordStrings[CHAR]:
	1046	return sqltypes.Char
	1047	case keywordStrings[VARCHAR]:
	1048	return sqltypes.VarChar
	1049	case keywordStrings[BINARY]:
	1050	return sqltypes.Binary
	1051	case keywordStrings[VARBINARY]:
	1052	return sqltypes.VarBinary
	1053	case keywordStrings[DATE]:
	1054	return sqltypes.Date
	1055	case keywordStrings[TIME]:
	1056	return sqltypes.Time
	1057	case keywordStrings[DATETIME]:
	1058	return sqltypes.Datetime
	1059	case keywordStrings[TIMESTAMP]:
	1060	return sqltypes.Timestamp
	1061	case keywordStrings[YEAR]:
	1062	return sqltypes.Year
	1063	case keywordStrings[FLOAT_TYPE]:
	1064	return sqltypes.Float32
	1065	case keywordStrings[DOUBLE]:
	1066	return sqltypes.Float64
	1067	case keywordStrings[DECIMAL]:
	1068	return sqltypes.Decimal
	1069	case keywordStrings[BIT]:
	1070	return sqltypes.Bit
	1071	case keywordStrings[ENUM]:
	1072	return sqltypes.Enum
	1073	case keywordStrings[SET]:
	1074	return sqltypes.Set
	1075	case keywordStrings[JSON]:
	1076	return sqltypes.TypeJSON
	1077	case keywordStrings[GEOMETRY]:
	1078	return sqltypes.Geometry
	1079	case keywordStrings[POINT]:
	1080	return sqltypes.Geometry
	1081	case keywordStrings[LINESTRING]:
	1082	return sqltypes.Geometry
	1083	case keywordStrings[POLYGON]:
	1084	return sqltypes.Geometry
	1085	case keywordStrings[GEOMETRYCOLLECTION]:
	1086	return sqltypes.Geometry
	1087	case keywordStrings[MULTIPOINT]:
	1088	return sqltypes.Geometry
	1089	case keywordStrings[MULTILINESTRING]:
	1090	return sqltypes.Geometry
	1091	case keywordStrings[MULTIPOLYGON]:
	1092	return sqltypes.Geometry
	1093	}
	1094	panic("unimplemented type " + ct.Type)
	1095	}
	1096
	1097	func (ct *ColumnType) walkSubtree(visit Visit) error {
	1098	return nil
	1099	}
	1100
	1101	// IndexDefinition describes an index in a CREATE TABLE statement
	1102	type IndexDefinition struct {
	1103	Info *IndexInfo
	1104	Columns []*IndexColumn
	1105	Options []*IndexOption
	1106	}
	1107
	1108	// Format formats the node.
	1109	func (idx IndexDefinition) Format(buf TrackedBuffer) {
	1110	buf.Myprintf("%v (", idx.Info)
	1111	for i, col := range idx.Columns {
	1112	if i != 0 {
	1113	buf.Myprintf(", %v", col.Column)
	1114	} else {
	1115	buf.Myprintf("%v", col.Column)
	1116	}
	1117	if col.Length != nil {
	1118	buf.Myprintf("(%v)", col.Length)
	1119	}
	1120	}
	1121	buf.Myprintf(")")
	1122
	1123	for _, opt := range idx.Options {
	1124	buf.Myprintf(" %s", opt.Name)
	1125	if opt.Using != "" {
	1126	buf.Myprintf(" %s", opt.Using)
	1127	} else {
	1128	buf.Myprintf(" %v", opt.Value)
	1129	}
	1130	}
	1131	}
	1132
	1133	func (idx *IndexDefinition) walkSubtree(visit Visit) error {
	1134	if idx == nil {
	1135	return nil
	1136	}
	1137
	1138	for _, n := range idx.Columns {
	1139	if err := Walk(visit, n.Column); err != nil {
	1140	return err
	1141	}
	1142	}
	1143
	1144	return nil
	1145	}
	1146
	1147	// IndexInfo describes the name and type of an index in a CREATE TABLE statement
	1148	type IndexInfo struct {
	1149	Type string
	1150	Name ColIdent
	1151	Primary bool
	1152	Spatial bool
	1153	Unique bool
	1154	}
	1155
	1156	// Format formats the node.
	1157	func (ii IndexInfo) Format(buf TrackedBuffer) {
	1158	if ii.Primary {
	1159	buf.Myprintf("%s", ii.Type)
	1160	} else {
	1161	buf.Myprintf("%s %v", ii.Type, ii.Name)
	1162	}
	1163	}
	1164
	1165	func (ii *IndexInfo) walkSubtree(visit Visit) error {
	1166	return Walk(visit, ii.Name)
	1167	}
	1168
	1169	// IndexColumn describes a column in an index definition with optional length
	1170	type IndexColumn struct {
	1171	Column ColIdent
	1172	Length *SQLVal
	1173	}
	1174
	1175	// LengthScaleOption is used for types that have an optional length
	1176	// and scale
	1177	type LengthScaleOption struct {
	1178	Length *SQLVal
	1179	Scale *SQLVal
	1180	}
	1181
	1182	// IndexOption is used for trailing options for indexes: COMMENT, KEY_BLOCK_SIZE, USING
	1183	type IndexOption struct {
	1184	Name string
	1185	Value *SQLVal
	1186	Using string
	1187	}
	1188
	1189	// ColumnKeyOption indicates whether or not the given column is defined as an
	1190	// index element and contains the type of the option
	1191	type ColumnKeyOption int
	1192
	1193	const (
	1194	colKeyNone ColumnKeyOption = iota
	1195	colKeyPrimary
	1196	colKeySpatialKey
	1197	colKeyUnique
	1198	colKeyUniqueKey
	1199	colKey
	1200	)
	1201
	1202	// VindexSpec defines a vindex for a CREATE VINDEX or DROP VINDEX statement
	1203	type VindexSpec struct {
	1204	Name ColIdent
	1205	Type ColIdent
	1206	Params []VindexParam
	1207	}
	1208
	1209	// ParseParams parses the vindex parameter list, pulling out the special-case
	1210	// "owner" parameter
	1211	func (node *VindexSpec) ParseParams() (string, map[string]string) {
	1212	var owner string
	1213	params := map[string]string{}
	1214	for _, p := range node.Params {
	1215	if p.Key.Lowered() == VindexOwnerStr {
	1216	owner = p.Val
	1217	} else {
	1218	params[p.Key.String()] = p.Val
	1219	}
	1220	}
	1221	return owner, params
	1222	}
	1223
	1224	// Format formats the node. The "CREATE VINDEX" preamble was formatted in
	1225	// the containing DDL node Format, so this just prints the type, any
	1226	// parameters, and optionally the owner
	1227	func (node VindexSpec) Format(buf TrackedBuffer) {
	1228	buf.Myprintf("using %v", node.Type)
	1229
	1230	numParams := len(node.Params)
	1231	if numParams != 0 {
	1232	buf.Myprintf(" with ")
	1233	for i, p := range node.Params {
	1234	if i != 0 {
	1235	buf.Myprintf(", ")
	1236	}
	1237	buf.Myprintf("%v", p)
	1238	}
	1239	}
	1240	}
	1241
	1242	func (node *VindexSpec) walkSubtree(visit Visit) error {
	1243	err := Walk(visit,
	1244	node.Name,
	1245	)
	1246
	1247	if err != nil {
	1248	return err
	1249	}
	1250
	1251	for _, p := range node.Params {
	1252	err := Walk(visit, p)
	1253
	1254	if err != nil {
	1255	return err
	1256	}
	1257	}
	1258	return nil
	1259	}
	1260
	1261	// VindexParam defines a key/value parameter for a CREATE VINDEX statement
	1262	type VindexParam struct {
	1263	Key ColIdent
	1264	Val string
	1265	}
	1266
	1267	// Format formats the node.
	1268	func (node VindexParam) Format(buf *TrackedBuffer) {
	1269	buf.Myprintf("%s=%s", node.Key.String(), node.Val)
	1270	}
	1271
	1272	func (node VindexParam) walkSubtree(visit Visit) error {
	1273	return Walk(visit,
	1274	node.Key,
	1275	)
	1276	}
	1277
	1278	// Show represents a show statement.
	1279	type Show struct {
	1280	Type string
	1281	OnTable TableName
	1282	ShowTablesOpt *ShowTablesOpt
	1283	Scope string
	1284	}
	1285
	1286	// Format formats the node.
	1287	func (node Show) Format(buf TrackedBuffer) {
	1288	if node.Type == "tables" && node.ShowTablesOpt != nil {
	1289	opt := node.ShowTablesOpt
	1290	if opt.DbName != "" {
	1291	if opt.Filter != nil {
	1292	buf.Myprintf("show %s%stables from %s %v", opt.Extended, opt.Full, opt.DbName, opt.Filter)
	1293	} else {
	1294	buf.Myprintf("show %s%stables from %s", opt.Extended, opt.Full, opt.DbName)
	1295	}
	1296	} else {
	1297	if opt.Filter != nil {
	1298	buf.Myprintf("show %s%stables %v", opt.Extended, opt.Full, opt.Filter)
	1299	} else {
	1300	buf.Myprintf("show %s%stables", opt.Extended, opt.Full)
	1301	}
	1302	}
	1303	return
	1304	}
	1305	if node.Scope == "" {
	1306	buf.Myprintf("show %s", node.Type)
	1307	} else {
	1308	buf.Myprintf("show %s %s", node.Scope, node.Type)
	1309	}
	1310	if node.HasOnTable() {
	1311	buf.Myprintf(" on %v", node.OnTable)
	1312	}
	1313	}
	1314
	1315	// HasOnTable returns true if the show statement has an "on" clause
	1316	func (node *Show) HasOnTable() bool {
	1317	return node.OnTable.Name.v != ""
	1318	}
	1319
	1320	func (node *Show) walkSubtree(visit Visit) error {
	1321	return nil
	1322	}
	1323
	1324	// ShowTablesOpt is show tables option
	1325	type ShowTablesOpt struct {
	1326	Extended string
	1327	Full string
	1328	DbName string
	1329	Filter *ShowFilter
	1330	}
	1331
	1332	// ShowFilter is show tables filter
	1333	type ShowFilter struct {
	1334	Like string
	1335	Filter Expr
	1336	}
	1337
	1338	// Format formats the node.
	1339	func (node ShowFilter) Format(buf TrackedBuffer) {
	1340	if node.Like != "" {
	1341	buf.Myprintf("like '%s'", node.Like)
	1342	} else {
	1343	buf.Myprintf("where %v", node.Filter)
	1344	}
	1345	}
	1346
	1347	func (node *ShowFilter) walkSubtree(visit Visit) error {
	1348	return nil
	1349	}
	1350
	1351	// Use represents a use statement.
	1352	type Use struct {
	1353	DBName TableIdent
	1354	}
	1355
	1356	// Format formats the node.
	1357	func (node Use) Format(buf TrackedBuffer) {
	1358	if node.DBName.v != "" {
	1359	buf.Myprintf("use %v", node.DBName)
	1360	} else {
	1361	buf.Myprintf("use")
	1362	}
	1363	}
	1364
	1365	func (node *Use) walkSubtree(visit Visit) error {
	1366	return Walk(visit, node.DBName)
	1367	}
	1368
	1369	// Begin represents a Begin statement.
	1370	type Begin struct{}
	1371
	1372	// Format formats the node.
	1373	func (node Begin) Format(buf TrackedBuffer) {
	1374	buf.WriteString("begin")
	1375	}
	1376
	1377	func (node *Begin) walkSubtree(visit Visit) error {
	1378	return nil
	1379	}
	1380
	1381	// Commit represents a Commit statement.
	1382	type Commit struct{}
	1383
	1384	// Format formats the node.
	1385	func (node Commit) Format(buf TrackedBuffer) {
	1386	buf.WriteString("commit")
	1387	}
	1388
	1389	func (node *Commit) walkSubtree(visit Visit) error {
	1390	return nil
	1391	}
	1392
	1393	// Rollback represents a Rollback statement.
	1394	type Rollback struct{}
	1395
	1396	// Format formats the node.
	1397	func (node Rollback) Format(buf TrackedBuffer) {
	1398	buf.WriteString("rollback")
	1399	}
	1400
	1401	func (node *Rollback) walkSubtree(visit Visit) error {
	1402	return nil
	1403	}
	1404
	1405	// OtherRead represents a DESCRIBE, or EXPLAIN statement.
	1406	// It should be used only as an indicator. It does not contain
	1407	// the full AST for the statement.
	1408	type OtherRead struct{}
	1409
	1410	// Format formats the node.
	1411	func (node OtherRead) Format(buf TrackedBuffer) {
	1412	buf.WriteString("otherread")
	1413	}
	1414
	1415	func (node *OtherRead) walkSubtree(visit Visit) error {
	1416	return nil
	1417	}
	1418
	1419	// OtherAdmin represents a misc statement that relies on ADMIN privileges,
	1420	// such as REPAIR, OPTIMIZE, or TRUNCATE statement.
	1421	// It should be used only as an indicator. It does not contain
	1422	// the full AST for the statement.
	1423	type OtherAdmin struct{}
	1424
	1425	// Format formats the node.
	1426	func (node OtherAdmin) Format(buf TrackedBuffer) {
	1427	buf.WriteString("otheradmin")
	1428	}
	1429
	1430	func (node *OtherAdmin) walkSubtree(visit Visit) error {
	1431	return nil
	1432	}
	1433
	1434	// Comments represents a list of comments.
	1435	type Comments [][]byte
	1436
	1437	// Format formats the node.
	1438	func (node Comments) Format(buf *TrackedBuffer) {
	1439	for _, c := range node {
	1440	buf.Myprintf("%s ", c)
	1441	}
	1442	}
	1443
	1444	func (node Comments) walkSubtree(visit Visit) error {
	1445	return nil
	1446	}
	1447
	1448	// SelectExprs represents SELECT expressions.
	1449	type SelectExprs []SelectExpr
	1450
	1451	// Format formats the node.
	1452	func (node SelectExprs) Format(buf *TrackedBuffer) {
	1453	var prefix string
	1454	for _, n := range node {
	1455	buf.Myprintf("%s%v", prefix, n)
	1456	prefix = ", "
	1457	}
	1458	}
	1459
	1460	func (node SelectExprs) walkSubtree(visit Visit) error {
	1461	for _, n := range node {
	1462	if err := Walk(visit, n); err != nil {
	1463	return err
	1464	}
	1465	}
	1466	return nil
	1467	}
	1468
	1469	// SelectExpr represents a SELECT expression.
	1470	type SelectExpr interface {
	1471	iSelectExpr()
	1472	SQLNode
	1473	}
	1474
	1475	func (*StarExpr) iSelectExpr() {}
	1476	func (*AliasedExpr) iSelectExpr() {}
	1477	func (Nextval) iSelectExpr() {}
	1478
	1479	// StarExpr defines a '' or 'table.' expression.
	1480	type StarExpr struct {
	1481	TableName TableName
	1482	}
	1483
	1484	// Format formats the node.
	1485	func (node StarExpr) Format(buf TrackedBuffer) {
	1486	if !node.TableName.IsEmpty() {
	1487	buf.Myprintf("%v.", node.TableName)
	1488	}
	1489	buf.Myprintf("*")
	1490	}
	1491
	1492	func (node *StarExpr) walkSubtree(visit Visit) error {
	1493	if node == nil {
	1494	return nil
	1495	}
	1496	return Walk(
	1497	visit,
	1498	node.TableName,
	1499	)
	1500	}
	1501
	1502	// AliasedExpr defines an aliased SELECT expression.
	1503	type AliasedExpr struct {
	1504	Expr Expr
	1505	As ColIdent
	1506	}
	1507
	1508	// Format formats the node.
	1509	func (node AliasedExpr) Format(buf TrackedBuffer) {
	1510	buf.Myprintf("%v", node.Expr)
	1511	if !node.As.IsEmpty() {
	1512	buf.Myprintf(" as %v", node.As)
	1513	}
	1514	}
	1515
	1516	func (node *AliasedExpr) walkSubtree(visit Visit) error {
	1517	if node == nil {
	1518	return nil
	1519	}
	1520	return Walk(
	1521	visit,
	1522	node.Expr,
	1523	node.As,
	1524	)
	1525	}
	1526
	1527	// Nextval defines the NEXT VALUE expression.
	1528	type Nextval struct {
	1529	Expr Expr
	1530	}
	1531
	1532	// Format formats the node.
	1533	func (node Nextval) Format(buf *TrackedBuffer) {
	1534	buf.Myprintf("next %v values", node.Expr)
	1535	}
	1536
	1537	func (node Nextval) walkSubtree(visit Visit) error {
	1538	return Walk(visit, node.Expr)
	1539	}
	1540
	1541	// Columns represents an insert column list.
	1542	type Columns []ColIdent
	1543
	1544	// Format formats the node.
	1545	func (node Columns) Format(buf *TrackedBuffer) {
	1546	if node == nil {
	1547	return
	1548	}
	1549	prefix := "("
	1550	for _, n := range node {
	1551	buf.Myprintf("%s%v", prefix, n)
	1552	prefix = ", "
	1553	}
	1554	buf.WriteString(")")
	1555	}
	1556
	1557	func (node Columns) walkSubtree(visit Visit) error {
	1558	for _, n := range node {
	1559	if err := Walk(visit, n); err != nil {
	1560	return err
	1561	}
	1562	}
	1563	return nil
	1564	}
	1565
	1566	// FindColumn finds a column in the column list, returning
	1567	// the index if it exists or -1 otherwise
	1568	func (node Columns) FindColumn(col ColIdent) int {
	1569	for i, colName := range node {
	1570	if colName.Equal(col) {
	1571	return i
	1572	}
	1573	}
	1574	return -1
	1575	}
	1576
	1577	// Partitions is a type alias for Columns so we can handle printing efficiently
	1578	type Partitions Columns
	1579
	1580	// Format formats the node
	1581	func (node Partitions) Format(buf *TrackedBuffer) {
	1582	if node == nil {
	1583	return
	1584	}
	1585	prefix := " partition ("
	1586	for _, n := range node {
	1587	buf.Myprintf("%s%v", prefix, n)
	1588	prefix = ", "
	1589	}
	1590	buf.WriteString(")")
	1591	}
	1592
	1593	func (node Partitions) walkSubtree(visit Visit) error {
	1594	for _, n := range node {
	1595	if err := Walk(visit, n); err != nil {
	1596	return err
	1597	}
	1598	}
	1599	return nil
	1600	}
	1601
	1602	// TableExprs represents a list of table expressions.
	1603	type TableExprs []TableExpr
	1604
	1605	// Format formats the node.
	1606	func (node TableExprs) Format(buf *TrackedBuffer) {
	1607	var prefix string
	1608	for _, n := range node {
	1609	buf.Myprintf("%s%v", prefix, n)
	1610	prefix = ", "
	1611	}
	1612	}
	1613
	1614	func (node TableExprs) walkSubtree(visit Visit) error {
	1615	for _, n := range node {
	1616	if err := Walk(visit, n); err != nil {
	1617	return err
	1618	}
	1619	}
	1620	return nil
	1621	}
	1622
	1623	// TableExpr represents a table expression.
	1624	type TableExpr interface {
	1625	iTableExpr()
	1626	SQLNode
	1627	}
	1628
	1629	func (*AliasedTableExpr) iTableExpr() {}
	1630	func (*ParenTableExpr) iTableExpr() {}
	1631	func (*JoinTableExpr) iTableExpr() {}
	1632
	1633	// AliasedTableExpr represents a table expression
	1634	// coupled with an optional alias or index hint.
	1635	// If As is empty, no alias was used.
	1636	type AliasedTableExpr struct {
	1637	Expr SimpleTableExpr
	1638	Partitions Partitions
	1639	As TableIdent
	1640	Hints *IndexHints
	1641	}
	1642
	1643	// Format formats the node.
	1644	func (node AliasedTableExpr) Format(buf TrackedBuffer) {
	1645	buf.Myprintf("%v%v", node.Expr, node.Partitions)
	1646	if !node.As.IsEmpty() {
	1647	buf.Myprintf(" as %v", node.As)
	1648	}
	1649	if node.Hints != nil {
	1650	// Hint node provides the space padding.
	1651	buf.Myprintf("%v", node.Hints)
	1652	}
	1653	}
	1654
	1655	func (node *AliasedTableExpr) walkSubtree(visit Visit) error {
	1656	if node == nil {
	1657	return nil
	1658	}
	1659	return Walk(
	1660	visit,
	1661	node.Expr,
	1662	node.As,
	1663	node.Hints,
	1664	)
	1665	}
	1666
	1667	// RemoveHints returns a new AliasedTableExpr with the hints removed.
	1668	func (node AliasedTableExpr) RemoveHints() AliasedTableExpr {
	1669	noHints := *node
	1670	noHints.Hints = nil
	1671	return &noHints
	1672	}
	1673
	1674	// SimpleTableExpr represents a simple table expression.
	1675	type SimpleTableExpr interface {
	1676	iSimpleTableExpr()
	1677	SQLNode
	1678	}
	1679
	1680	func (TableName) iSimpleTableExpr() {}
	1681	func (*Subquery) iSimpleTableExpr() {}
	1682
	1683	// TableNames is a list of TableName.
	1684	type TableNames []TableName
	1685
	1686	// Format formats the node.
	1687	func (node TableNames) Format(buf *TrackedBuffer) {
	1688	var prefix string
	1689	for _, n := range node {
	1690	buf.Myprintf("%s%v", prefix, n)
	1691	prefix = ", "
	1692	}
	1693	}
	1694
	1695	func (node TableNames) walkSubtree(visit Visit) error {
	1696	for _, n := range node {
	1697	if err := Walk(visit, n); err != nil {
	1698	return err
	1699	}
	1700	}
	1701	return nil
	1702	}
	1703
	1704	// TableName represents a table name.
	1705	// Qualifier, if specified, represents a database or keyspace.
	1706	// TableName is a value struct whose fields are case sensitive.
	1707	// This means two TableName vars can be compared for equality
	1708	// and a TableName can also be used as key in a map.
	1709	type TableName struct {
	1710	Name, Qualifier TableIdent
	1711	}
	1712
	1713	// Format formats the node.
	1714	func (node TableName) Format(buf *TrackedBuffer) {
	1715	if node.IsEmpty() {
	1716	return
	1717	}
	1718	if !node.Qualifier.IsEmpty() {
	1719	buf.Myprintf("%v.", node.Qualifier)
	1720	}
	1721	buf.Myprintf("%v", node.Name)
	1722	}
	1723
	1724	func (node TableName) walkSubtree(visit Visit) error {
	1725	return Walk(
	1726	visit,
	1727	node.Name,
	1728	node.Qualifier,
	1729	)
	1730	}
	1731
	1732	// IsEmpty returns true if TableName is nil or empty.
	1733	func (node TableName) IsEmpty() bool {
	1734	// If Name is empty, Qualifer is also empty.
	1735	return node.Name.IsEmpty()
	1736	}
	1737
	1738	// ToViewName returns a TableName acceptable for use as a VIEW. VIEW names are
	1739	// always lowercase, so ToViewName lowercasese the name. Databases are case-sensitive
	1740	// so Qualifier is left untouched.
	1741	func (node TableName) ToViewName() TableName {
	1742	return TableName{
	1743	Qualifier: node.Qualifier,
	1744	Name: NewTableIdent(strings.ToLower(node.Name.v)),
	1745	}
	1746	}
	1747
	1748	// ParenTableExpr represents a parenthesized list of TableExpr.
	1749	type ParenTableExpr struct {
	1750	Exprs TableExprs
	1751	}
	1752
	1753	// Format formats the node.
	1754	func (node ParenTableExpr) Format(buf TrackedBuffer) {
	1755	buf.Myprintf("(%v)", node.Exprs)
	1756	}
	1757
	1758	func (node *ParenTableExpr) walkSubtree(visit Visit) error {
	1759	if node == nil {
	1760	return nil
	1761	}
	1762	return Walk(
	1763	visit,
	1764	node.Exprs,
	1765	)
	1766	}
	1767
	1768	// JoinCondition represents the join conditions (either a ON or USING clause)
	1769	// of a JoinTableExpr.
	1770	type JoinCondition struct {
	1771	On Expr
	1772	Using Columns
	1773	}
	1774
	1775	// Format formats the node.
	1776	func (node JoinCondition) Format(buf *TrackedBuffer) {
	1777	if node.On != nil {
	1778	buf.Myprintf(" on %v", node.On)
	1779	}
	1780	if node.Using != nil {
	1781	buf.Myprintf(" using %v", node.Using)
	1782	}
	1783	}
	1784
	1785	func (node JoinCondition) walkSubtree(visit Visit) error {
	1786	return Walk(
	1787	visit,
	1788	node.On,
	1789	node.Using,
	1790	)
	1791	}
	1792
	1793	// JoinTableExpr represents a TableExpr that's a JOIN operation.
	1794	type JoinTableExpr struct {
	1795	LeftExpr TableExpr
	1796	Join string
	1797	RightExpr TableExpr
	1798	Condition JoinCondition
	1799	}
	1800
	1801	// JoinTableExpr.Join
	1802	const (
	1803	JoinStr = "join"
	1804	StraightJoinStr = "straight_join"
	1805	LeftJoinStr = "left join"
	1806	RightJoinStr = "right join"
	1807	NaturalJoinStr = "natural join"
	1808	NaturalLeftJoinStr = "natural left join"
	1809	NaturalRightJoinStr = "natural right join"
	1810	)
	1811
	1812	// Format formats the node.
	1813	func (node JoinTableExpr) Format(buf TrackedBuffer) {
	1814	buf.Myprintf("%v %s %v%v", node.LeftExpr, node.Join, node.RightExpr, node.Condition)
	1815	}
	1816
	1817	func (node *JoinTableExpr) walkSubtree(visit Visit) error {
	1818	if node == nil {
	1819	return nil
	1820	}
	1821	return Walk(
	1822	visit,
	1823	node.LeftExpr,
	1824	node.RightExpr,
	1825	node.Condition,
	1826	)
	1827	}
	1828
	1829	// IndexHints represents a list of index hints.
	1830	type IndexHints struct {
	1831	Type string
	1832	Indexes []ColIdent
	1833	}
	1834
	1835	// Index hints.
	1836	const (
	1837	UseStr = "use "
	1838	IgnoreStr = "ignore "
	1839	ForceStr = "force "
	1840	)
	1841
	1842	// Format formats the node.
	1843	func (node IndexHints) Format(buf TrackedBuffer) {
	1844	buf.Myprintf(" %sindex ", node.Type)
	1845	prefix := "("
	1846	for _, n := range node.Indexes {
	1847	buf.Myprintf("%s%v", prefix, n)
	1848	prefix = ", "
	1849	}
	1850	buf.Myprintf(")")
	1851	}
	1852
	1853	func (node *IndexHints) walkSubtree(visit Visit) error {
	1854	if node == nil {
	1855	return nil
	1856	}
	1857	for _, n := range node.Indexes {
	1858	if err := Walk(visit, n); err != nil {
	1859	return err
	1860	}
	1861	}
	1862	return nil
	1863	}
	1864
	1865	// Where represents a WHERE or HAVING clause.
	1866	type Where struct {
	1867	Type string
	1868	Expr Expr
	1869	}
	1870
	1871	// Where.Type
	1872	const (
	1873	WhereStr = "where"
	1874	HavingStr = "having"
	1875	)
	1876
	1877	// NewWhere creates a WHERE or HAVING clause out
	1878	// of a Expr. If the expression is nil, it returns nil.
	1879	func NewWhere(typ string, expr Expr) *Where {
	1880	if expr == nil {
	1881	return nil
	1882	}
	1883	return &Where{Type: typ, Expr: expr}
	1884	}
	1885
	1886	// Format formats the node.
	1887	func (node Where) Format(buf TrackedBuffer) {
	1888	if node == nil \|\| node.Expr == nil {
	1889	return
	1890	}
	1891	buf.Myprintf(" %s %v", node.Type, node.Expr)
	1892	}
	1893
	1894	func (node *Where) walkSubtree(visit Visit) error {
	1895	if node == nil {
	1896	return nil
	1897	}
	1898	return Walk(
	1899	visit,
	1900	node.Expr,
	1901	)
	1902	}
	1903
	1904	// Expr represents an expression.
	1905	type Expr interface {
	1906	iExpr()
	1907	// replace replaces any subexpression that matches
	1908	// from with to. The implementation can use the
	1909	// replaceExprs convenience function.
	1910	replace(from, to Expr) bool
	1911	SQLNode
	1912	}
	1913
	1914	func (*AndExpr) iExpr() {}
	1915	func (*OrExpr) iExpr() {}
	1916	func (*NotExpr) iExpr() {}
	1917	func (*ParenExpr) iExpr() {}
	1918	func (*ComparisonExpr) iExpr() {}
	1919	func (*RangeCond) iExpr() {}
	1920	func (*IsExpr) iExpr() {}
	1921	func (*ExistsExpr) iExpr() {}
	1922	func (*SQLVal) iExpr() {}
	1923	func (*NullVal) iExpr() {}
	1924	func (BoolVal) iExpr() {}
	1925	func (*ColName) iExpr() {}
	1926	func (ValTuple) iExpr() {}
	1927	func (*Subquery) iExpr() {}
	1928	func (ListArg) iExpr() {}
	1929	func (*BinaryExpr) iExpr() {}
	1930	func (*UnaryExpr) iExpr() {}
	1931	func (*IntervalExpr) iExpr() {}
	1932	func (*CollateExpr) iExpr() {}
	1933	func (*FuncExpr) iExpr() {}
	1934	func (*CaseExpr) iExpr() {}
	1935	func (*ValuesFuncExpr) iExpr() {}
	1936	func (*ConvertExpr) iExpr() {}
	1937	func (*SubstrExpr) iExpr() {}
	1938	func (*ConvertUsingExpr) iExpr() {}
	1939	func (*MatchExpr) iExpr() {}
	1940	func (*GroupConcatExpr) iExpr() {}
	1941	func (*Default) iExpr() {}
	1942
	1943	// ReplaceExpr finds the from expression from root
	1944	// and replaces it with to. If from matches root,
	1945	// then to is returned.
	1946	func ReplaceExpr(root, from, to Expr) Expr {
	1947	if root == from {
	1948	return to
	1949	}
	1950	root.replace(from, to)
	1951	return root
	1952	}
	1953
	1954	// replaceExprs is a convenience function used by implementors
	1955	// of the replace method.
	1956	func replaceExprs(from, to Expr, exprs ...*Expr) bool {
	1957	for _, expr := range exprs {
	1958	if *expr == nil {
	1959	continue
	1960	}
	1961	if *expr == from {
	1962	*expr = to
	1963	return true
	1964	}
	1965	if (*expr).replace(from, to) {
	1966	return true
	1967	}
	1968	}
	1969	return false
	1970	}
	1971
	1972	// Exprs represents a list of value expressions.
	1973	// It's not a valid expression because it's not parenthesized.
	1974	type Exprs []Expr
	1975
	1976	// Format formats the node.
	1977	func (node Exprs) Format(buf *TrackedBuffer) {
	1978	var prefix string
	1979	for _, n := range node {
	1980	buf.Myprintf("%s%v", prefix, n)
	1981	prefix = ", "
	1982	}
	1983	}
	1984
	1985	func (node Exprs) walkSubtree(visit Visit) error {
	1986	for _, n := range node {
	1987	if err := Walk(visit, n); err != nil {
	1988	return err
	1989	}
	1990	}
	1991	return nil
	1992	}
	1993
	1994	// AndExpr represents an AND expression.
	1995	type AndExpr struct {
	1996	Left, Right Expr
	1997	}
	1998
	1999	// Format formats the node.
	2000	func (node AndExpr) Format(buf TrackedBuffer) {
	2001	buf.Myprintf("%v and %v", node.Left, node.Right)
	2002	}
	2003
	2004	func (node *AndExpr) walkSubtree(visit Visit) error {
	2005	if node == nil {
	2006	return nil
	2007	}
	2008	return Walk(
	2009	visit,
	2010	node.Left,
	2011	node.Right,
	2012	)
	2013	}
	2014
	2015	func (node *AndExpr) replace(from, to Expr) bool {
	2016	return replaceExprs(from, to, &node.Left, &node.Right)
	2017	}
	2018
	2019	// OrExpr represents an OR expression.
	2020	type OrExpr struct {
	2021	Left, Right Expr
	2022	}
	2023
	2024	// Format formats the node.
	2025	func (node OrExpr) Format(buf TrackedBuffer) {
	2026	buf.Myprintf("%v or %v", node.Left, node.Right)
	2027	}
	2028
	2029	func (node *OrExpr) walkSubtree(visit Visit) error {
	2030	if node == nil {
	2031	return nil
	2032	}
	2033	return Walk(
	2034	visit,
	2035	node.Left,
	2036	node.Right,
	2037	)
	2038	}
	2039
	2040	func (node *OrExpr) replace(from, to Expr) bool {
	2041	return replaceExprs(from, to, &node.Left, &node.Right)
	2042	}
	2043
	2044	// NotExpr represents a NOT expression.
	2045	type NotExpr struct {
	2046	Expr Expr
	2047	}
	2048
	2049	// Format formats the node.
	2050	func (node NotExpr) Format(buf TrackedBuffer) {
	2051	buf.Myprintf("not %v", node.Expr)
	2052	}
	2053
	2054	func (node *NotExpr) walkSubtree(visit Visit) error {
	2055	if node == nil {
	2056	return nil
	2057	}
	2058	return Walk(
	2059	visit,
	2060	node.Expr,
	2061	)
	2062	}
	2063
	2064	func (node *NotExpr) replace(from, to Expr) bool {
	2065	return replaceExprs(from, to, &node.Expr)
	2066	}
	2067
	2068	// ParenExpr represents a parenthesized boolean expression.
	2069	type ParenExpr struct {
	2070	Expr Expr
	2071	}
	2072
	2073	// Format formats the node.
	2074	func (node ParenExpr) Format(buf TrackedBuffer) {
	2075	buf.Myprintf("(%v)", node.Expr)
	2076	}
	2077
	2078	func (node *ParenExpr) walkSubtree(visit Visit) error {
	2079	if node == nil {
	2080	return nil
	2081	}
	2082	return Walk(
	2083	visit,
	2084	node.Expr,
	2085	)
	2086	}
	2087
	2088	func (node *ParenExpr) replace(from, to Expr) bool {
	2089	return replaceExprs(from, to, &node.Expr)
	2090	}
	2091
	2092	// ComparisonExpr represents a two-value comparison expression.
	2093	type ComparisonExpr struct {
	2094	Operator string
	2095	Left, Right Expr
	2096	Escape Expr
	2097	}
	2098
	2099	// ComparisonExpr.Operator
	2100	const (
	2101	EqualStr = "="
	2102	LessThanStr = "<"
	2103	GreaterThanStr = ">"
	2104	LessEqualStr = "<="
	2105	GreaterEqualStr = ">="
	2106	NotEqualStr = "!="
	2107	NullSafeEqualStr = "<=>"
	2108	InStr = "in"
	2109	NotInStr = "not in"
	2110	LikeStr = "like"
	2111	NotLikeStr = "not like"
	2112	RegexpStr = "regexp"
	2113	NotRegexpStr = "not regexp"
	2114	JSONExtractOp = "->"
	2115	JSONUnquoteExtractOp = "->>"
	2116	)
	2117
	2118	// Format formats the node.
	2119	func (node ComparisonExpr) Format(buf TrackedBuffer) {
	2120	buf.Myprintf("%v %s %v", node.Left, node.Operator, node.Right)
	2121	if node.Escape != nil {
	2122	buf.Myprintf(" escape %v", node.Escape)
	2123	}
	2124	}
	2125
	2126	func (node *ComparisonExpr) walkSubtree(visit Visit) error {
	2127	if node == nil {
	2128	return nil
	2129	}
	2130	return Walk(
	2131	visit,
	2132	node.Left,
	2133	node.Right,
	2134	node.Escape,
	2135	)
	2136	}
	2137
	2138	func (node *ComparisonExpr) replace(from, to Expr) bool {
	2139	return replaceExprs(from, to, &node.Left, &node.Right, &node.Escape)
	2140	}
	2141
	2142	// RangeCond represents a BETWEEN or a NOT BETWEEN expression.
	2143	type RangeCond struct {
	2144	Operator string
	2145	Left Expr
	2146	From, To Expr
	2147	}
	2148
	2149	// RangeCond.Operator
	2150	const (
	2151	BetweenStr = "between"
	2152	NotBetweenStr = "not between"
	2153	)
	2154
	2155	// Format formats the node.
	2156	func (node RangeCond) Format(buf TrackedBuffer) {
	2157	buf.Myprintf("%v %s %v and %v", node.Left, node.Operator, node.From, node.To)
	2158	}
	2159
	2160	func (node *RangeCond) walkSubtree(visit Visit) error {
	2161	if node == nil {
	2162	return nil
	2163	}
	2164	return Walk(
	2165	visit,
	2166	node.Left,
	2167	node.From,
	2168	node.To,
	2169	)
	2170	}
	2171
	2172	func (node *RangeCond) replace(from, to Expr) bool {
	2173	return replaceExprs(from, to, &node.Left, &node.From, &node.To)
	2174	}
	2175
	2176	// IsExpr represents an IS ... or an IS NOT ... expression.
	2177	type IsExpr struct {
	2178	Operator string
	2179	Expr Expr
	2180	}
	2181
	2182	// IsExpr.Operator
	2183	const (
	2184	IsNullStr = "is null"
	2185	IsNotNullStr = "is not null"
	2186	IsTrueStr = "is true"
	2187	IsNotTrueStr = "is not true"
	2188	IsFalseStr = "is false"
	2189	IsNotFalseStr = "is not false"
	2190	)
	2191
	2192	// Format formats the node.
	2193	func (node IsExpr) Format(buf TrackedBuffer) {
	2194	buf.Myprintf("%v %s", node.Expr, node.Operator)
	2195	}
	2196
	2197	func (node *IsExpr) walkSubtree(visit Visit) error {
	2198	if node == nil {
	2199	return nil
	2200	}
	2201	return Walk(
	2202	visit,
	2203	node.Expr,
	2204	)
	2205	}
	2206
	2207	func (node *IsExpr) replace(from, to Expr) bool {
	2208	return replaceExprs(from, to, &node.Expr)
	2209	}
	2210
	2211	// ExistsExpr represents an EXISTS expression.
	2212	type ExistsExpr struct {
	2213	Subquery *Subquery
	2214	}
	2215
	2216	// Format formats the node.
	2217	func (node ExistsExpr) Format(buf TrackedBuffer) {
	2218	buf.Myprintf("exists %v", node.Subquery)
	2219	}
	2220
	2221	func (node *ExistsExpr) walkSubtree(visit Visit) error {
	2222	if node == nil {
	2223	return nil
	2224	}
	2225	return Walk(
	2226	visit,
	2227	node.Subquery,
	2228	)
	2229	}
	2230
	2231	func (node *ExistsExpr) replace(from, to Expr) bool {
	2232	return false
	2233	}
	2234
	2235	// ExprFromValue converts the given Value into an Expr or returns an error.
	2236	func ExprFromValue(value sqltypes.Value) (Expr, error) {
	2237	// The type checks here follow the rules defined in sqltypes/types.go.
	2238	switch {
	2239	case value.Type() == sqltypes.Null:
	2240	return &NullVal{}, nil
	2241	case value.IsIntegral():
	2242	return NewIntVal(value.ToBytes()), nil
	2243	case value.IsFloat() \|\| value.Type() == sqltypes.Decimal:
	2244	return NewFloatVal(value.ToBytes()), nil
	2245	case value.IsQuoted():
	2246	return NewStrVal(value.ToBytes()), nil
	2247	default:
	2248	// We cannot support sqltypes.Expression, or any other invalid type.
	2249	return nil, fmt.Errorf("cannot convert value %v to AST", value)
	2250	}
	2251	}
	2252
	2253	// ValType specifies the type for SQLVal.
	2254	type ValType int
	2255
	2256	// These are the possible Valtype values.
	2257	// HexNum represents a 0x... value. It cannot
	2258	// be treated as a simple value because it can
	2259	// be interpreted differently depending on the
	2260	// context.
	2261	const (
	2262	StrVal = ValType(iota)
	2263	IntVal
	2264	FloatVal
	2265	HexNum
	2266	HexVal
	2267	ValArg
	2268	BitVal
	2269	)
	2270
	2271	// SQLVal represents a single value.
	2272	type SQLVal struct {
	2273	Type ValType
	2274	Val []byte
	2275	}
	2276
	2277	// NewStrVal builds a new StrVal.
	2278	func NewStrVal(in []byte) *SQLVal {
	2279	return &SQLVal{Type: StrVal, Val: in}
	2280	}
	2281
	2282	// NewIntVal builds a new IntVal.
	2283	func NewIntVal(in []byte) *SQLVal {
	2284	return &SQLVal{Type: IntVal, Val: in}
	2285	}
	2286
	2287	// NewFloatVal builds a new FloatVal.
	2288	func NewFloatVal(in []byte) *SQLVal {
	2289	return &SQLVal{Type: FloatVal, Val: in}
	2290	}
	2291
	2292	// NewHexNum builds a new HexNum.
	2293	func NewHexNum(in []byte) *SQLVal {
	2294	return &SQLVal{Type: HexNum, Val: in}
	2295	}
	2296
	2297	// NewHexVal builds a new HexVal.
	2298	func NewHexVal(in []byte) *SQLVal {
	2299	return &SQLVal{Type: HexVal, Val: in}
	2300	}
	2301
	2302	// NewBitVal builds a new BitVal containing a bit literal.
	2303	func NewBitVal(in []byte) *SQLVal {
	2304	return &SQLVal{Type: BitVal, Val: in}
	2305	}
	2306
	2307	// NewValArg builds a new ValArg.
	2308	func NewValArg(in []byte) *SQLVal {
	2309	return &SQLVal{Type: ValArg, Val: in}
	2310	}
	2311
	2312	// Format formats the node.
	2313	func (node SQLVal) Format(buf TrackedBuffer) {
	2314	switch node.Type {
	2315	case StrVal:
	2316	sqltypes.MakeTrusted(sqltypes.VarBinary, node.Val).EncodeSQL(buf)
	2317	case IntVal, FloatVal, HexNum:
	2318	buf.Myprintf("%s", []byte(node.Val))
	2319	case HexVal:
	2320	buf.Myprintf("X'%s'", []byte(node.Val))
	2321	case BitVal:
	2322	buf.Myprintf("B'%s'", []byte(node.Val))
	2323	case ValArg:
	2324	buf.WriteArg(string(node.Val))
	2325	default:
	2326	panic("unexpected")
	2327	}
	2328	}
	2329
	2330	func (node *SQLVal) walkSubtree(visit Visit) error {
	2331	return nil
	2332	}
	2333
	2334	func (node *SQLVal) replace(from, to Expr) bool {
	2335	return false
	2336	}
	2337
	2338	// HexDecode decodes the hexval into bytes.
	2339	func (node *SQLVal) HexDecode() ([]byte, error) {
	2340	dst := make([]byte, hex.DecodedLen(len([]byte(node.Val))))
	2341	_, err := hex.Decode(dst, []byte(node.Val))
	2342	if err != nil {
	2343	return nil, err
	2344	}
	2345	return dst, err
	2346	}
	2347
	2348	// NullVal represents a NULL value.
	2349	type NullVal struct{}
	2350
	2351	// Format formats the node.
	2352	func (node NullVal) Format(buf TrackedBuffer) {
	2353	buf.Myprintf("null")
	2354	}
	2355
	2356	func (node *NullVal) walkSubtree(visit Visit) error {
	2357	return nil
	2358	}
	2359
	2360	func (node *NullVal) replace(from, to Expr) bool {
	2361	return false
	2362	}
	2363
	2364	// BoolVal is true or false.
	2365	type BoolVal bool
	2366
	2367	// Format formats the node.
	2368	func (node BoolVal) Format(buf *TrackedBuffer) {
	2369	if node {
	2370	buf.Myprintf("true")
	2371	} else {
	2372	buf.Myprintf("false")
	2373	}
	2374	}
	2375
	2376	func (node BoolVal) walkSubtree(visit Visit) error {
	2377	return nil
	2378	}
	2379
	2380	func (node BoolVal) replace(from, to Expr) bool {
	2381	return false
	2382	}
	2383
	2384	// ColName represents a column name.
	2385	type ColName struct {
	2386	// Metadata is not populated by the parser.
	2387	// It's a placeholder for analyzers to store
	2388	// additional data, typically info about which
	2389	// table or column this node references.
	2390	Metadata interface{}
	2391	Name ColIdent
	2392	Qualifier TableName
	2393	}
	2394
	2395	// Format formats the node.
	2396	func (node ColName) Format(buf TrackedBuffer) {
	2397	if !node.Qualifier.IsEmpty() {
	2398	buf.Myprintf("%v.", node.Qualifier)
	2399	}
	2400	buf.Myprintf("%v", node.Name)
	2401	}
	2402
	2403	func (node *ColName) walkSubtree(visit Visit) error {
	2404	if node == nil {
	2405	return nil
	2406	}
	2407	return Walk(
	2408	visit,
	2409	node.Name,
	2410	node.Qualifier,
	2411	)
	2412	}
	2413
	2414	func (node *ColName) replace(from, to Expr) bool {
	2415	return false
	2416	}
	2417
	2418	// Equal returns true if the column names match.
	2419	func (node ColName) Equal(c ColName) bool {
	2420	// Failsafe: ColName should not be empty.
	2421	if node == nil \|\| c == nil {
	2422	return false
	2423	}
	2424	return node.Name.Equal(c.Name) && node.Qualifier == c.Qualifier
	2425	}
	2426
	2427	// ColTuple represents a list of column values.
	2428	// It can be ValTuple, Subquery, ListArg.
	2429	type ColTuple interface {
	2430	iColTuple()
	2431	Expr
	2432	}
	2433
	2434	func (ValTuple) iColTuple() {}
	2435	func (*Subquery) iColTuple() {}
	2436	func (ListArg) iColTuple() {}
	2437
	2438	// ValTuple represents a tuple of actual values.
	2439	type ValTuple Exprs
	2440
	2441	// Format formats the node.
	2442	func (node ValTuple) Format(buf *TrackedBuffer) {
	2443	buf.Myprintf("(%v)", Exprs(node))
	2444	}
	2445
	2446	func (node ValTuple) walkSubtree(visit Visit) error {
	2447	return Walk(visit, Exprs(node))
	2448	}
	2449
	2450	func (node ValTuple) replace(from, to Expr) bool {
	2451	for i := range node {
	2452	if replaceExprs(from, to, &node[i]) {
	2453	return true
	2454	}
	2455	}
	2456	return false
	2457	}
	2458
	2459	// Subquery represents a subquery.
	2460	type Subquery struct {
	2461	Select SelectStatement
	2462	}
	2463
	2464	// Format formats the node.
	2465	func (node Subquery) Format(buf TrackedBuffer) {
	2466	buf.Myprintf("(%v)", node.Select)
	2467	}
	2468
	2469	func (node *Subquery) walkSubtree(visit Visit) error {
	2470	if node == nil {
	2471	return nil
	2472	}
	2473	return Walk(
	2474	visit,
	2475	node.Select,
	2476	)
	2477	}
	2478
	2479	func (node *Subquery) replace(from, to Expr) bool {
	2480	return false
	2481	}
	2482
	2483	// ListArg represents a named list argument.
	2484	type ListArg []byte
	2485
	2486	// Format formats the node.
	2487	func (node ListArg) Format(buf *TrackedBuffer) {
	2488	buf.WriteArg(string(node))
	2489	}
	2490
	2491	func (node ListArg) walkSubtree(visit Visit) error {
	2492	return nil
	2493	}
	2494
	2495	func (node ListArg) replace(from, to Expr) bool {
	2496	return false
	2497	}
	2498
	2499	// BinaryExpr represents a binary value expression.
	2500	type BinaryExpr struct {
	2501	Operator string
	2502	Left, Right Expr
	2503	}
	2504
	2505	// BinaryExpr.Operator
	2506	const (
	2507	BitAndStr = "&"
	2508	BitOrStr = "\|"
	2509	BitXorStr = "^"
	2510	PlusStr = "+"
	2511	MinusStr = "-"
	2512	MultStr = "*"
	2513	DivStr = "/"
	2514	IntDivStr = "div"
	2515	ModStr = "%"
	2516	ShiftLeftStr = "<<"
	2517	ShiftRightStr = ">>"
	2518	)
	2519
	2520	// Format formats the node.
	2521	func (node BinaryExpr) Format(buf TrackedBuffer) {
	2522	buf.Myprintf("%v %s %v", node.Left, node.Operator, node.Right)
	2523	}
	2524
	2525	func (node *BinaryExpr) walkSubtree(visit Visit) error {
	2526	if node == nil {
	2527	return nil
	2528	}
	2529	return Walk(
	2530	visit,
	2531	node.Left,
	2532	node.Right,
	2533	)
	2534	}
	2535
	2536	func (node *BinaryExpr) replace(from, to Expr) bool {
	2537	return replaceExprs(from, to, &node.Left, &node.Right)
	2538	}
	2539
	2540	// UnaryExpr represents a unary value expression.
	2541	type UnaryExpr struct {
	2542	Operator string
	2543	Expr Expr
	2544	}
	2545
	2546	// UnaryExpr.Operator
	2547	const (
	2548	UPlusStr = "+"
	2549	UMinusStr = "-"
	2550	TildaStr = "~"
	2551	BangStr = "!"
	2552	BinaryStr = "binary "
	2553	UBinaryStr = "_binary "
	2554	)
	2555
	2556	// Format formats the node.
	2557	func (node UnaryExpr) Format(buf TrackedBuffer) {
	2558	if _, unary := node.Expr.(*UnaryExpr); unary {
	2559	buf.Myprintf("%s %v", node.Operator, node.Expr)
	2560	return
	2561	}
	2562	buf.Myprintf("%s%v", node.Operator, node.Expr)
	2563	}
	2564
	2565	func (node *UnaryExpr) walkSubtree(visit Visit) error {
	2566	if node == nil {
	2567	return nil
	2568	}
	2569	return Walk(
	2570	visit,
	2571	node.Expr,
	2572	)
	2573	}
	2574
	2575	func (node *UnaryExpr) replace(from, to Expr) bool {
	2576	return replaceExprs(from, to, &node.Expr)
	2577	}
	2578
	2579	// IntervalExpr represents a date-time INTERVAL expression.
	2580	type IntervalExpr struct {
	2581	Expr Expr
	2582	Unit string
	2583	}
	2584
	2585	// Format formats the node.
	2586	func (node IntervalExpr) Format(buf TrackedBuffer) {
	2587	buf.Myprintf("interval %v %s", node.Expr, node.Unit)
	2588	}
	2589
	2590	func (node *IntervalExpr) walkSubtree(visit Visit) error {
	2591	if node == nil {
	2592	return nil
	2593	}
	2594	return Walk(
	2595	visit,
	2596	node.Expr,
	2597	)
	2598	}
	2599
	2600	func (node *IntervalExpr) replace(from, to Expr) bool {
	2601	return replaceExprs(from, to, &node.Expr)
	2602	}
	2603
	2604	// CollateExpr represents dynamic collate operator.
	2605	type CollateExpr struct {
	2606	Expr Expr
	2607	Charset string
	2608	}
	2609
	2610	// Format formats the node.
	2611	func (node CollateExpr) Format(buf TrackedBuffer) {
	2612	buf.Myprintf("%v collate %s", node.Expr, node.Charset)
	2613	}
	2614
	2615	func (node *CollateExpr) walkSubtree(visit Visit) error {
	2616	if node == nil {
	2617	return nil
	2618	}
	2619	return Walk(
	2620	visit,
	2621	node.Expr,
	2622	)
	2623	}
	2624
	2625	func (node *CollateExpr) replace(from, to Expr) bool {
	2626	return replaceExprs(from, to, &node.Expr)
	2627	}
	2628
	2629	// FuncExpr represents a function call.
	2630	type FuncExpr struct {
	2631	Qualifier TableIdent
	2632	Name ColIdent
	2633	Distinct bool
	2634	Exprs SelectExprs
	2635	}
	2636
	2637	// Format formats the node.
	2638	func (node FuncExpr) Format(buf TrackedBuffer) {
	2639	var distinct string
	2640	if node.Distinct {
	2641	distinct = "distinct "
	2642	}
	2643	if !node.Qualifier.IsEmpty() {
	2644	buf.Myprintf("%v.", node.Qualifier)
	2645	}
	2646	// Function names should not be back-quoted even
	2647	// if they match a reserved word. So, print the
	2648	// name as is.
	2649	buf.Myprintf("%s(%s%v)", node.Name.String(), distinct, node.Exprs)
	2650	}
	2651
	2652	func (node *FuncExpr) walkSubtree(visit Visit) error {
	2653	if node == nil {
	2654	return nil
	2655	}
	2656	return Walk(
	2657	visit,
	2658	node.Qualifier,
	2659	node.Name,
	2660	node.Exprs,
	2661	)
	2662	}
	2663
	2664	func (node *FuncExpr) replace(from, to Expr) bool {
	2665	for _, sel := range node.Exprs {
	2666	aliased, ok := sel.(*AliasedExpr)
	2667	if !ok {
	2668	continue
	2669	}
	2670	if replaceExprs(from, to, &aliased.Expr) {
	2671	return true
	2672	}
	2673	}
	2674	return false
	2675	}
	2676
	2677	// Aggregates is a map of all aggregate functions.
	2678	var Aggregates = map[string]bool{
	2679	"avg": true,
	2680	"bit_and": true,
	2681	"bit_or": true,
	2682	"bit_xor": true,
	2683	"count": true,
	2684	"group_concat": true,
	2685	"max": true,
	2686	"min": true,
	2687	"std": true,
	2688	"stddev_pop": true,
	2689	"stddev_samp": true,
	2690	"stddev": true,
	2691	"sum": true,
	2692	"var_pop": true,
	2693	"var_samp": true,
	2694	"variance": true,
	2695	}
	2696
	2697	// IsAggregate returns true if the function is an aggregate.
	2698	func (node *FuncExpr) IsAggregate() bool {
	2699	return Aggregates[node.Name.Lowered()]
	2700	}
	2701
	2702	// GroupConcatExpr represents a call to GROUP_CONCAT
	2703	type GroupConcatExpr struct {
	2704	Distinct string
	2705	Exprs SelectExprs
	2706	OrderBy OrderBy
	2707	Separator string
	2708	}
	2709
	2710	// Format formats the node
	2711	func (node GroupConcatExpr) Format(buf TrackedBuffer) {
	2712	buf.Myprintf("group_concat(%s%v%v%s)", node.Distinct, node.Exprs, node.OrderBy, node.Separator)
	2713	}
	2714
	2715	func (node *GroupConcatExpr) walkSubtree(visit Visit) error {
	2716	if node == nil {
	2717	return nil
	2718	}
	2719	return Walk(
	2720	visit,
	2721	node.Exprs,
	2722	node.OrderBy,
	2723	)
	2724	}
	2725
	2726	func (node *GroupConcatExpr) replace(from, to Expr) bool {
	2727	for _, sel := range node.Exprs {
	2728	aliased, ok := sel.(*AliasedExpr)
	2729	if !ok {
	2730	continue
	2731	}
	2732	if replaceExprs(from, to, &aliased.Expr) {
	2733	return true
	2734	}
	2735	}
	2736	for _, order := range node.OrderBy {
	2737	if replaceExprs(from, to, &order.Expr) {
	2738	return true
	2739	}
	2740	}
	2741	return false
	2742	}
	2743
	2744	// ValuesFuncExpr represents a function call.
	2745	type ValuesFuncExpr struct {
	2746	Name *ColName
	2747	}
	2748
	2749	// Format formats the node.
	2750	func (node ValuesFuncExpr) Format(buf TrackedBuffer) {
	2751	buf.Myprintf("values(%v)", node.Name)
	2752	}
	2753
	2754	func (node *ValuesFuncExpr) walkSubtree(visit Visit) error {
	2755	if node == nil {
	2756	return nil
	2757	}
	2758	return Walk(
	2759	visit,
	2760	node.Name,
	2761	)
	2762	}
	2763
	2764	func (node *ValuesFuncExpr) replace(from, to Expr) bool {
	2765	return false
	2766	}
	2767
	2768	// SubstrExpr represents a call to SubstrExpr(column, value_expression) or SubstrExpr(column, value_expression,value_expression)
	2769	// also supported syntax SubstrExpr(column from value_expression for value_expression)
	2770	type SubstrExpr struct {
	2771	Name *ColName
	2772	From Expr
	2773	To Expr
	2774	}
	2775
	2776	// Format formats the node.
	2777	func (node SubstrExpr) Format(buf TrackedBuffer) {
	2778
	2779	if node.To == nil {
	2780	buf.Myprintf("substr(%v, %v)", node.Name, node.From)
	2781	} else {
	2782	buf.Myprintf("substr(%v, %v, %v)", node.Name, node.From, node.To)
	2783	}
	2784	}
	2785
	2786	func (node *SubstrExpr) replace(from, to Expr) bool {
	2787	return replaceExprs(from, to, &node.From, &node.To)
	2788	}
	2789
	2790	func (node *SubstrExpr) walkSubtree(visit Visit) error {
	2791	if node == nil {
	2792	return nil
	2793	}
	2794	return Walk(
	2795	visit,
	2796	node.Name,
	2797	node.From,
	2798	node.To,
	2799	)
	2800	}
	2801
	2802	// ConvertExpr represents a call to CONVERT(expr, type)
	2803	// or it's equivalent CAST(expr AS type). Both are rewritten to the former.
	2804	type ConvertExpr struct {
	2805	Expr Expr
	2806	Type *ConvertType
	2807	}
	2808
	2809	// Format formats the node.
	2810	func (node ConvertExpr) Format(buf TrackedBuffer) {
	2811	buf.Myprintf("convert(%v, %v)", node.Expr, node.Type)
	2812	}
	2813
	2814	func (node *ConvertExpr) walkSubtree(visit Visit) error {
	2815	if node == nil {
	2816	return nil
	2817	}
	2818	return Walk(
	2819	visit,
	2820	node.Expr,
	2821	node.Type,
	2822	)
	2823	}
	2824
	2825	func (node *ConvertExpr) replace(from, to Expr) bool {
	2826	return replaceExprs(from, to, &node.Expr)
	2827	}
	2828
	2829	// ConvertUsingExpr represents a call to CONVERT(expr USING charset).
	2830	type ConvertUsingExpr struct {
	2831	Expr Expr
	2832	Type string
	2833	}
	2834
	2835	// Format formats the node.
	2836	func (node ConvertUsingExpr) Format(buf TrackedBuffer) {
	2837	buf.Myprintf("convert(%v using %s)", node.Expr, node.Type)
	2838	}
	2839
	2840	func (node *ConvertUsingExpr) walkSubtree(visit Visit) error {
	2841	if node == nil {
	2842	return nil
	2843	}
	2844	return Walk(
	2845	visit,
	2846	node.Expr,
	2847	)
	2848	}
	2849
	2850	func (node *ConvertUsingExpr) replace(from, to Expr) bool {
	2851	return replaceExprs(from, to, &node.Expr)
	2852	}
	2853
	2854	// ConvertType represents the type in call to CONVERT(expr, type)
	2855	type ConvertType struct {
	2856	Type string
	2857	Length *SQLVal
	2858	Scale *SQLVal
	2859	Operator string
	2860	Charset string
	2861	}
	2862
	2863	// this string is "character set" and this comment is required
	2864	const (
	2865	CharacterSetStr = " character set"
	2866	)
	2867
	2868	// Format formats the node.
	2869	func (node ConvertType) Format(buf TrackedBuffer) {
	2870	buf.Myprintf("%s", node.Type)
	2871	if node.Length != nil {
	2872	buf.Myprintf("(%v", node.Length)
	2873	if node.Scale != nil {
	2874	buf.Myprintf(", %v", node.Scale)
	2875	}
	2876	buf.Myprintf(")")
	2877	}
	2878	if node.Charset != "" {
	2879	buf.Myprintf("%s %s", node.Operator, node.Charset)
	2880	}
	2881	}
	2882
	2883	func (node *ConvertType) walkSubtree(visit Visit) error {
	2884	return nil
	2885	}
	2886
	2887	// MatchExpr represents a call to the MATCH function
	2888	type MatchExpr struct {
	2889	Columns SelectExprs
	2890	Expr Expr
	2891	Option string
	2892	}
	2893
	2894	// MatchExpr.Option
	2895	const (
	2896	BooleanModeStr = " in boolean mode"
	2897	NaturalLanguageModeStr = " in natural language mode"
	2898	NaturalLanguageModeWithQueryExpansionStr = " in natural language mode with query expansion"
	2899	QueryExpansionStr = " with query expansion"
	2900	)
	2901
	2902	// Format formats the node
	2903	func (node MatchExpr) Format(buf TrackedBuffer) {
	2904	buf.Myprintf("match(%v) against (%v%s)", node.Columns, node.Expr, node.Option)
	2905	}
	2906
	2907	func (node *MatchExpr) walkSubtree(visit Visit) error {
	2908	if node == nil {
	2909	return nil
	2910	}
	2911	return Walk(
	2912	visit,
	2913	node.Columns,
	2914	node.Expr,
	2915	)
	2916	}
	2917
	2918	func (node *MatchExpr) replace(from, to Expr) bool {
	2919	for _, sel := range node.Columns {
	2920	aliased, ok := sel.(*AliasedExpr)
	2921	if !ok {
	2922	continue
	2923	}
	2924	if replaceExprs(from, to, &aliased.Expr) {
	2925	return true
	2926	}
	2927	}
	2928	return replaceExprs(from, to, &node.Expr)
	2929	}
	2930
	2931	// CaseExpr represents a CASE expression.
	2932	type CaseExpr struct {
	2933	Expr Expr
	2934	Whens []*When
	2935	Else Expr
	2936	}
	2937
	2938	// Format formats the node.
	2939	func (node CaseExpr) Format(buf TrackedBuffer) {
	2940	buf.Myprintf("case ")
	2941	if node.Expr != nil {
	2942	buf.Myprintf("%v ", node.Expr)
	2943	}
	2944	for _, when := range node.Whens {
	2945	buf.Myprintf("%v ", when)
	2946	}
	2947	if node.Else != nil {
	2948	buf.Myprintf("else %v ", node.Else)
	2949	}
	2950	buf.Myprintf("end")
	2951	}
	2952
	2953	func (node *CaseExpr) walkSubtree(visit Visit) error {
	2954	if node == nil {
	2955	return nil
	2956	}
	2957	if err := Walk(visit, node.Expr); err != nil {
	2958	return err
	2959	}
	2960	for _, n := range node.Whens {
	2961	if err := Walk(visit, n); err != nil {
	2962	return err
	2963	}
	2964	}
	2965	return Walk(visit, node.Else)
	2966	}
	2967
	2968	func (node *CaseExpr) replace(from, to Expr) bool {
	2969	for _, when := range node.Whens {
	2970	if replaceExprs(from, to, &when.Cond, &when.Val) {
	2971	return true
	2972	}
	2973	}
	2974	return replaceExprs(from, to, &node.Expr, &node.Else)
	2975	}
	2976
	2977	// Default represents a DEFAULT expression.
	2978	type Default struct {
	2979	ColName string
	2980	}
	2981
	2982	// Format formats the node.
	2983	func (node Default) Format(buf TrackedBuffer) {
	2984	buf.Myprintf("default")
	2985	if node.ColName != "" {
	2986	buf.Myprintf("(%s)", node.ColName)
	2987	}
	2988	}
	2989
	2990	func (node *Default) walkSubtree(visit Visit) error {
	2991	return nil
	2992	}
	2993
	2994	func (node *Default) replace(from, to Expr) bool {
	2995	return false
	2996	}
	2997
	2998	// When represents a WHEN sub-expression.
	2999	type When struct {
	3000	Cond Expr
	3001	Val Expr
	3002	}
	3003
	3004	// Format formats the node.
	3005	func (node When) Format(buf TrackedBuffer) {
	3006	buf.Myprintf("when %v then %v", node.Cond, node.Val)
	3007	}
	3008
	3009	func (node *When) walkSubtree(visit Visit) error {
	3010	if node == nil {
	3011	return nil
	3012	}
	3013	return Walk(
	3014	visit,
	3015	node.Cond,
	3016	node.Val,
	3017	)
	3018	}
	3019
	3020	// GroupBy represents a GROUP BY clause.
	3021	type GroupBy []Expr
	3022
	3023	// Format formats the node.
	3024	func (node GroupBy) Format(buf *TrackedBuffer) {
	3025	prefix := " group by "
	3026	for _, n := range node {
	3027	buf.Myprintf("%s%v", prefix, n)
	3028	prefix = ", "
	3029	}
	3030	}
	3031
	3032	func (node GroupBy) walkSubtree(visit Visit) error {
	3033	for _, n := range node {
	3034	if err := Walk(visit, n); err != nil {
	3035	return err
	3036	}
	3037	}
	3038	return nil
	3039	}
	3040
	3041	// OrderBy represents an ORDER By clause.
	3042	type OrderBy []*Order
	3043
	3044	// Format formats the node.
	3045	func (node OrderBy) Format(buf *TrackedBuffer) {
	3046	prefix := " order by "
	3047	for _, n := range node {
	3048	buf.Myprintf("%s%v", prefix, n)
	3049	prefix = ", "
	3050	}
	3051	}
	3052
	3053	func (node OrderBy) walkSubtree(visit Visit) error {
	3054	for _, n := range node {
	3055	if err := Walk(visit, n); err != nil {
	3056	return err
	3057	}
	3058	}
	3059	return nil
	3060	}
	3061
	3062	// Order represents an ordering expression.
	3063	type Order struct {
	3064	Expr Expr
	3065	Direction string
	3066	}
	3067
	3068	// Order.Direction
	3069	const (
	3070	AscScr = "asc"
	3071	DescScr = "desc"
	3072	)
	3073
	3074	// Format formats the node.
	3075	func (node Order) Format(buf TrackedBuffer) {
	3076	if node, ok := node.Expr.(*NullVal); ok {
	3077	buf.Myprintf("%v", node)
	3078	return
	3079	}
	3080	if node, ok := node.Expr.(*FuncExpr); ok {
	3081	if node.Name.Lowered() == "rand" {
	3082	buf.Myprintf("%v", node)
	3083	return
	3084	}
	3085	}
	3086
	3087	buf.Myprintf("%v %s", node.Expr, node.Direction)
	3088	}
	3089
	3090	func (node *Order) walkSubtree(visit Visit) error {
	3091	if node == nil {
	3092	return nil
	3093	}
	3094	return Walk(
	3095	visit,
	3096	node.Expr,
	3097	)
	3098	}
	3099
	3100	// Limit represents a LIMIT clause.
	3101	type Limit struct {
	3102	Offset, Rowcount Expr
	3103	}
	3104
	3105	// Format formats the node.
	3106	func (node Limit) Format(buf TrackedBuffer) {
	3107	if node == nil {
	3108	return
	3109	}
	3110	buf.Myprintf(" limit ")
	3111	if node.Offset != nil {
	3112	buf.Myprintf("%v, ", node.Offset)
	3113	}
	3114	buf.Myprintf("%v", node.Rowcount)
	3115	}
	3116
	3117	func (node *Limit) walkSubtree(visit Visit) error {
	3118	if node == nil {
	3119	return nil
	3120	}
	3121	return Walk(
	3122	visit,
	3123	node.Offset,
	3124	node.Rowcount,
	3125	)
	3126	}
	3127
	3128	// Values represents a VALUES clause.
	3129	type Values []ValTuple
	3130
	3131	// Format formats the node.
	3132	func (node Values) Format(buf *TrackedBuffer) {
	3133	prefix := "values "
	3134	for _, n := range node {
	3135	buf.Myprintf("%s%v", prefix, n)
	3136	prefix = ", "
	3137	}
	3138	}
	3139
	3140	func (node Values) walkSubtree(visit Visit) error {
	3141	for _, n := range node {
	3142	if err := Walk(visit, n); err != nil {
	3143	return err
	3144	}
	3145	}
	3146	return nil
	3147	}
	3148
	3149	// UpdateExprs represents a list of update expressions.
	3150	type UpdateExprs []*UpdateExpr
	3151
	3152	// Format formats the node.
	3153	func (node UpdateExprs) Format(buf *TrackedBuffer) {
	3154	var prefix string
	3155	for _, n := range node {
	3156	buf.Myprintf("%s%v", prefix, n)
	3157	prefix = ", "
	3158	}
	3159	}
	3160
	3161	func (node UpdateExprs) walkSubtree(visit Visit) error {
	3162	for _, n := range node {
	3163	if err := Walk(visit, n); err != nil {
	3164	return err
	3165	}
	3166	}
	3167	return nil
	3168	}
	3169
	3170	// UpdateExpr represents an update expression.
	3171	type UpdateExpr struct {
	3172	Name *ColName
	3173	Expr Expr
	3174	}
	3175
	3176	// Format formats the node.
	3177	func (node UpdateExpr) Format(buf TrackedBuffer) {
	3178	buf.Myprintf("%v = %v", node.Name, node.Expr)
	3179	}
	3180
	3181	func (node *UpdateExpr) walkSubtree(visit Visit) error {
	3182	if node == nil {
	3183	return nil
	3184	}
	3185	return Walk(
	3186	visit,
	3187	node.Name,
	3188	node.Expr,
	3189	)
	3190	}
	3191
	3192	// SetExprs represents a list of set expressions.
	3193	type SetExprs []*SetExpr
	3194
	3195	// Format formats the node.
	3196	func (node SetExprs) Format(buf *TrackedBuffer) {
	3197	var prefix string
	3198	for _, n := range node {
	3199	buf.Myprintf("%s%v", prefix, n)
	3200	prefix = ", "
	3201	}
	3202	}
	3203
	3204	func (node SetExprs) walkSubtree(visit Visit) error {
	3205	for _, n := range node {
	3206	if err := Walk(visit, n); err != nil {
	3207	return err
	3208	}
	3209	}
	3210	return nil
	3211	}
	3212
	3213	// SetExpr represents a set expression.
	3214	type SetExpr struct {
	3215	Name ColIdent
	3216	Expr Expr
	3217	}
	3218
	3219	// Format formats the node.
	3220	func (node SetExpr) Format(buf TrackedBuffer) {
	3221	// We don't have to backtick set variable names.
	3222	if node.Name.EqualString("charset") \|\| node.Name.EqualString("names") {
	3223	buf.Myprintf("%s %v", node.Name.String(), node.Expr)
	3224	} else {
	3225	buf.Myprintf("%s = %v", node.Name.String(), node.Expr)
	3226	}
	3227	}
	3228
	3229	func (node *SetExpr) walkSubtree(visit Visit) error {
	3230	if node == nil {
	3231	return nil
	3232	}
	3233	return Walk(
	3234	visit,
	3235	node.Name,
	3236	node.Expr,
	3237	)
	3238	}
	3239
	3240	// OnDup represents an ON DUPLICATE KEY clause.
	3241	type OnDup UpdateExprs
	3242
	3243	// Format formats the node.
	3244	func (node OnDup) Format(buf *TrackedBuffer) {
	3245	if node == nil {
	3246	return
	3247	}
	3248	buf.Myprintf(" on duplicate key update %v", UpdateExprs(node))
	3249	}
	3250
	3251	func (node OnDup) walkSubtree(visit Visit) error {
	3252	return Walk(visit, UpdateExprs(node))
	3253	}
	3254
	3255	// ColIdent is a case insensitive SQL identifier. It will be escaped with
	3256	// backquotes if necessary.
	3257	type ColIdent struct {
	3258	// This artifact prevents this struct from being compared
	3259	// with itself. It consumes no space as long as it's not the
	3260	// last field in the struct.
	3261	_ [0]struct{ _ []byte }
	3262	val, lowered string
	3263	}
	3264
	3265	// NewColIdent makes a new ColIdent.
	3266	func NewColIdent(str string) ColIdent {
	3267	return ColIdent{
	3268	val: str,
	3269	}
	3270	}
	3271
	3272	// Format formats the node.
	3273	func (node ColIdent) Format(buf *TrackedBuffer) {
	3274	formatID(buf, node.val, node.Lowered())
	3275	}
	3276
	3277	func (node ColIdent) walkSubtree(visit Visit) error {
	3278	return nil
	3279	}
	3280
	3281	// IsEmpty returns true if the name is empty.
	3282	func (node ColIdent) IsEmpty() bool {
	3283	return node.val == ""
	3284	}
	3285
	3286	// String returns the unescaped column name. It must
	3287	// not be used for SQL generation. Use sqlparser.String
	3288	// instead. The Stringer conformance is for usage
	3289	// in templates.
	3290	func (node ColIdent) String() string {
	3291	return node.val
	3292	}
	3293
	3294	// CompliantName returns a compliant id name
	3295	// that can be used for a bind var.
	3296	func (node ColIdent) CompliantName() string {
	3297	return compliantName(node.val)
	3298	}
	3299
	3300	// Lowered returns a lower-cased column name.
	3301	// This function should generally be used only for optimizing
	3302	// comparisons.
	3303	func (node ColIdent) Lowered() string {
	3304	if node.val == "" {
	3305	return ""
	3306	}
	3307	if node.lowered == "" {
	3308	node.lowered = strings.ToLower(node.val)
	3309	}
	3310	return node.lowered
	3311	}
	3312
	3313	// Equal performs a case-insensitive compare.
	3314	func (node ColIdent) Equal(in ColIdent) bool {
	3315	return node.Lowered() == in.Lowered()
	3316	}
	3317
	3318	// EqualString performs a case-insensitive compare with str.
	3319	func (node ColIdent) EqualString(str string) bool {
	3320	return node.Lowered() == strings.ToLower(str)
	3321	}
	3322
	3323	// MarshalJSON marshals into JSON.
	3324	func (node ColIdent) MarshalJSON() ([]byte, error) {
	3325	return json.Marshal(node.val)
	3326	}
	3327
	3328	// UnmarshalJSON unmarshals from JSON.
	3329	func (node *ColIdent) UnmarshalJSON(b []byte) error {
	3330	var result string
	3331	err := json.Unmarshal(b, &result)
	3332	if err != nil {
	3333	return err
	3334	}
	3335	node.val = result
	3336	return nil
	3337	}
	3338
	3339	// TableIdent is a case sensitive SQL identifier. It will be escaped with
	3340	// backquotes if necessary.
	3341	type TableIdent struct {
	3342	v string
	3343	}
	3344
	3345	// NewTableIdent creates a new TableIdent.
	3346	func NewTableIdent(str string) TableIdent {
	3347	return TableIdent{v: str}
	3348	}
	3349
	3350	// Format formats the node.
	3351	func (node TableIdent) Format(buf *TrackedBuffer) {
	3352	formatID(buf, node.v, strings.ToLower(node.v))
	3353	}
	3354
	3355	func (node TableIdent) walkSubtree(visit Visit) error {
	3356	return nil
	3357	}
	3358
	3359	// IsEmpty returns true if TabIdent is empty.
	3360	func (node TableIdent) IsEmpty() bool {
	3361	return node.v == ""
	3362	}
	3363
	3364	// String returns the unescaped table name. It must
	3365	// not be used for SQL generation. Use sqlparser.String
	3366	// instead. The Stringer conformance is for usage
	3367	// in templates.
	3368	func (node TableIdent) String() string {
	3369	return node.v
	3370	}
	3371
	3372	// CompliantName returns a compliant id name
	3373	// that can be used for a bind var.
	3374	func (node TableIdent) CompliantName() string {
	3375	return compliantName(node.v)
	3376	}
	3377
	3378	// MarshalJSON marshals into JSON.
	3379	func (node TableIdent) MarshalJSON() ([]byte, error) {
	3380	return json.Marshal(node.v)
	3381	}
	3382
	3383	// UnmarshalJSON unmarshals from JSON.
	3384	func (node *TableIdent) UnmarshalJSON(b []byte) error {
	3385	var result string
	3386	err := json.Unmarshal(b, &result)
	3387	if err != nil {
	3388	return err
	3389	}
	3390	node.v = result
	3391	return nil
	3392	}
	3393
	3394	// Backtick produces a backticked literal given an input string.
	3395	func Backtick(in string) string {
	3396	var buf bytes.Buffer
	3397	buf.WriteByte('`')
	3398	for _, c := range in {
	3399	buf.WriteRune(c)
	3400	if c == '`' {
	3401	buf.WriteByte('`')
	3402	}
	3403	}
	3404	buf.WriteByte('`')
	3405	return buf.String()
	3406	}
	3407
	3408	func formatID(buf *TrackedBuffer, original, lowered string) {
	3409	isDbSystemVariable := false
	3410	if len(original) > 1 && original[:2] == "@@" {
	3411	isDbSystemVariable = true
	3412	}
	3413
	3414	for i, c := range original {
	3415	if !isLetter(uint16(c)) && (!isDbSystemVariable \|\| !isCarat(uint16(c))) {
	3416	if i == 0 \|\| !isDigit(uint16(c)) {
	3417	goto mustEscape
	3418	}
	3419	}
	3420	}
	3421	if _, ok := keywords[lowered]; ok {
	3422	goto mustEscape
	3423	}
	3424	buf.Myprintf("%s", original)
	3425	return
	3426
	3427	mustEscape:
	3428	buf.WriteByte('`')
	3429	for _, c := range original {
	3430	buf.WriteRune(c)
	3431	if c == '`' {
	3432	buf.WriteByte('`')
	3433	}
	3434	}
	3435	buf.WriteByte('`')
	3436	}
	3437
	3438	func compliantName(in string) string {
	3439	var buf bytes.Buffer
	3440	for i, c := range in {
	3441	if !isLetter(uint16(c)) {
	3442	if i == 0 \|\| !isDigit(uint16(c)) {
	3443	buf.WriteByte('_')
	3444	continue
	3445	}
	3446	}
	3447	buf.WriteRune(c)
	3448	}
	3449	return buf.String()
	3450	}

vendor/github.com/xwb1989/sqlparser/comments.go+293−0

@@ -0,0 +1,293 @@
	1	/*
	2	Copyright 2017 Google Inc.
	3
	4	Licensed under the Apache License, Version 2.0 (the "License");
	5	you may not use this file except in compliance with the License.
	6	You may obtain a copy of the License at
	7
	8	http://www.apache.org/licenses/LICENSE-2.0
	9
	10	Unless required by applicable law or agreed to in writing, software
	11	distributed under the License is distributed on an "AS IS" BASIS,
	12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	See the License for the specific language governing permissions and
	14	limitations under the License.
	15	*/
	16
	17	package sqlparser
	18
	19	import (
	20	"strconv"
	21	"strings"
	22	"unicode"
	23	)
	24
	25	const (
	26	// DirectiveMultiShardAutocommit is the query comment directive to allow
	27	// single round trip autocommit with a multi-shard statement.
	28	DirectiveMultiShardAutocommit = "MULTI_SHARD_AUTOCOMMIT"
	29	// DirectiveSkipQueryPlanCache skips query plan cache when set.
	30	DirectiveSkipQueryPlanCache = "SKIP_QUERY_PLAN_CACHE"
	31	// DirectiveQueryTimeout sets a query timeout in vtgate. Only supported for SELECTS.
	32	DirectiveQueryTimeout = "QUERY_TIMEOUT_MS"
	33	)
	34
	35	func isNonSpace(r rune) bool {
	36	return !unicode.IsSpace(r)
	37	}
	38
	39	// leadingCommentEnd returns the first index after all leading comments, or
	40	// 0 if there are no leading comments.
	41	func leadingCommentEnd(text string) (end int) {
	42	hasComment := false
	43	pos := 0
	44	for pos < len(text) {
	45	// Eat up any whitespace. Trailing whitespace will be considered part of
	46	// the leading comments.
	47	nextVisibleOffset := strings.IndexFunc(text[pos:], isNonSpace)
	48	if nextVisibleOffset < 0 {
	49	break
	50	}
	51	pos += nextVisibleOffset
	52	remainingText := text[pos:]
	53
	54	// Found visible characters. Look for '/*' at the beginning
	55	// and '*/' somewhere after that.
	56	if len(remainingText) < 4 \|\| remainingText[:2] != "/*" {
	57	break
	58	}
	59	commentLength := 4 + strings.Index(remainingText[2:], "*/")
	60	if commentLength < 4 {
	61	// Missing end comment :/
	62	break
	63	}
	64
	65	hasComment = true
	66	pos += commentLength
	67	}
	68
	69	if hasComment {
	70	return pos
	71	}
	72	return 0
	73	}
	74
	75	// trailingCommentStart returns the first index of trailing comments.
	76	// If there are no trailing comments, returns the length of the input string.
	77	func trailingCommentStart(text string) (start int) {
	78	hasComment := false
	79	reducedLen := len(text)
	80	for reducedLen > 0 {
	81	// Eat up any whitespace. Leading whitespace will be considered part of
	82	// the trailing comments.
	83	nextReducedLen := strings.LastIndexFunc(text[:reducedLen], isNonSpace) + 1
	84	if nextReducedLen == 0 {
	85	break
	86	}
	87	reducedLen = nextReducedLen
	88	if reducedLen < 4 \|\| text[reducedLen-2:reducedLen] != "*/" {
	89	break
	90	}
	91
	92	// Find the beginning of the comment
	93	startCommentPos := strings.LastIndex(text[:reducedLen-2], "/*")
	94	if startCommentPos < 0 {
	95	// Badly formatted sql :/
	96	break
	97	}
	98
	99	hasComment = true
	100	reducedLen = startCommentPos
	101	}
	102
	103	if hasComment {
	104	return reducedLen
	105	}
	106	return len(text)
	107	}
	108
	109	// MarginComments holds the leading and trailing comments that surround a query.
	110	type MarginComments struct {
	111	Leading string
	112	Trailing string
	113	}
	114
	115	// SplitMarginComments pulls out any leading or trailing comments from a raw sql query.
	116	// This function also trims leading (if there's a comment) and trailing whitespace.
	117	func SplitMarginComments(sql string) (query string, comments MarginComments) {
	118	trailingStart := trailingCommentStart(sql)
	119	leadingEnd := leadingCommentEnd(sql[:trailingStart])
	120	comments = MarginComments{
	121	Leading: strings.TrimLeftFunc(sql[:leadingEnd], unicode.IsSpace),
	122	Trailing: strings.TrimRightFunc(sql[trailingStart:], unicode.IsSpace),
	123	}
	124	return strings.TrimFunc(sql[leadingEnd:trailingStart], unicode.IsSpace), comments
	125	}
	126
	127	// StripLeadingComments trims the SQL string and removes any leading comments
	128	func StripLeadingComments(sql string) string {
	129	sql = strings.TrimFunc(sql, unicode.IsSpace)
	130
	131	for hasCommentPrefix(sql) {
	132	switch sql[0] {
	133	case '/':
	134	// Multi line comment
	135	index := strings.Index(sql, "*/")
	136	if index <= 1 {
	137	return sql
	138	}
	139	// don't strip /! ... / or /!50700 ... /
	140	if len(sql) > 2 && sql[2] == '!' {
	141	return sql
	142	}
	143	sql = sql[index+2:]
	144	case '-':
	145	// Single line comment
	146	index := strings.Index(sql, "\n")
	147	if index == -1 {
	148	return sql
	149	}
	150	sql = sql[index+1:]
	151	}
	152
	153	sql = strings.TrimFunc(sql, unicode.IsSpace)
	154	}
	155
	156	return sql
	157	}
	158
	159	func hasCommentPrefix(sql string) bool {
	160	return len(sql) > 1 && ((sql[0] == '/' && sql[1] == '*') \|\| (sql[0] == '-' && sql[1] == '-'))
	161	}
	162
	163	// ExtractMysqlComment extracts the version and SQL from a comment-only query
	164	// such as /!50708 sql here /
	165	func ExtractMysqlComment(sql string) (version string, innerSQL string) {
	166	sql = sql[3 : len(sql)-2]
	167
	168	digitCount := 0
	169	endOfVersionIndex := strings.IndexFunc(sql, func(c rune) bool {
	170	digitCount++
	171	return !unicode.IsDigit(c) \|\| digitCount == 6
	172	})
	173	version = sql[0:endOfVersionIndex]
	174	innerSQL = strings.TrimFunc(sql[endOfVersionIndex:], unicode.IsSpace)
	175
	176	return version, innerSQL
	177	}
	178
	179	const commentDirectivePreamble = "/*vt+"
	180
	181	// CommentDirectives is the parsed representation for execution directives
	182	// conveyed in query comments
	183	type CommentDirectives map[string]interface{}
	184
	185	// ExtractCommentDirectives parses the comment list for any execution directives
	186	// of the form:
	187	//
	188	// /vt+ OPTION_ONE=1 OPTION_TWO OPTION_THREE=abcd /
	189	//
	190	// It returns the map of the directive values or nil if there aren't any.
	191	func ExtractCommentDirectives(comments Comments) CommentDirectives {
	192	if comments == nil {
	193	return nil
	194	}
	195
	196	var vals map[string]interface{}
	197
	198	for _, comment := range comments {
	199	commentStr := string(comment)
	200	if commentStr[0:5] != commentDirectivePreamble {
	201	continue
	202	}
	203
	204	if vals == nil {
	205	vals = make(map[string]interface{})
	206	}
	207
	208	// Split on whitespace and ignore the first and last directive
	209	// since they contain the comment start/end
	210	directives := strings.Fields(commentStr)
	211	for i := 1; i < len(directives)-1; i++ {
	212	directive := directives[i]
	213	sep := strings.IndexByte(directive, '=')
	214
	215	// No value is equivalent to a true boolean
	216	if sep == -1 {
	217	vals[directive] = true
	218	continue
	219	}
	220
	221	strVal := directive[sep+1:]
	222	directive = directive[:sep]
	223
	224	intVal, err := strconv.Atoi(strVal)
	225	if err == nil {
	226	vals[directive] = intVal
	227	continue
	228	}
	229
	230	boolVal, err := strconv.ParseBool(strVal)
	231	if err == nil {
	232	vals[directive] = boolVal
	233	continue
	234	}
	235
	236	vals[directive] = strVal
	237	}
	238	}
	239	return vals
	240	}
	241
	242	// IsSet checks the directive map for the named directive and returns
	243	// true if the directive is set and has a true/false or 0/1 value
	244	func (d CommentDirectives) IsSet(key string) bool {
	245	if d == nil {
	246	return false
	247	}
	248
	249	val, ok := d[key]
	250	if !ok {
	251	return false
	252	}
	253
	254	boolVal, ok := val.(bool)
	255	if ok {
	256	return boolVal
	257	}
	258
	259	intVal, ok := val.(int)
	260	if ok {
	261	return intVal == 1
	262	}
	263	return false
	264	}
	265
	266	// SkipQueryPlanCacheDirective returns true if skip query plan cache directive is set to true in query.
	267	func SkipQueryPlanCacheDirective(stmt Statement) bool {
	268	switch stmt := stmt.(type) {
	269	case *Select:
	270	directives := ExtractCommentDirectives(stmt.Comments)
	271	if directives.IsSet(DirectiveSkipQueryPlanCache) {
	272	return true
	273	}
	274	case *Insert:
	275	directives := ExtractCommentDirectives(stmt.Comments)
	276	if directives.IsSet(DirectiveSkipQueryPlanCache) {
	277	return true
	278	}
	279	case *Update:
	280	directives := ExtractCommentDirectives(stmt.Comments)
	281	if directives.IsSet(DirectiveSkipQueryPlanCache) {
	282	return true
	283	}
	284	case *Delete:
	285	directives := ExtractCommentDirectives(stmt.Comments)
	286	if directives.IsSet(DirectiveSkipQueryPlanCache) {
	287	return true
	288	}
	289	default:
	290	return false
	291	}
	292	return false
	293	}

vendor/github.com/xwb1989/sqlparser/dependency/bytes2/buffer.go+65−0

@@ -0,0 +1,65 @@
	1	/*
	2	Copyright 2017 Google Inc.
	3
	4	Licensed under the Apache License, Version 2.0 (the "License");
	5	you may not use this file except in compliance with the License.
	6	You may obtain a copy of the License at
	7
	8	http://www.apache.org/licenses/LICENSE-2.0
	9
	10	Unless required by applicable law or agreed to in writing, software
	11	distributed under the License is distributed on an "AS IS" BASIS,
	12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	See the License for the specific language governing permissions and
	14	limitations under the License.
	15	*/
	16
	17	package bytes2
	18
	19	// Buffer implements a subset of the write portion of
	20	// bytes.Buffer, but more efficiently. This is meant to
	21	// be used in very high QPS operations, especially for
	22	// WriteByte, and without abstracting it as a Writer.
	23	// Function signatures contain errors for compatibility,
	24	// but they do not return errors.
	25	type Buffer struct {
	26	bytes []byte
	27	}
	28
	29	// NewBuffer is equivalent to bytes.NewBuffer.
	30	func NewBuffer(b []byte) *Buffer {
	31	return &Buffer{bytes: b}
	32	}
	33
	34	// Write is equivalent to bytes.Buffer.Write.
	35	func (buf *Buffer) Write(b []byte) (int, error) {
	36	buf.bytes = append(buf.bytes, b...)
	37	return len(b), nil
	38	}
	39
	40	// WriteString is equivalent to bytes.Buffer.WriteString.
	41	func (buf *Buffer) WriteString(s string) (int, error) {
	42	buf.bytes = append(buf.bytes, s...)
	43	return len(s), nil
	44	}
	45
	46	// WriteByte is equivalent to bytes.Buffer.WriteByte.
	47	func (buf *Buffer) WriteByte(b byte) error {
	48	buf.bytes = append(buf.bytes, b)
	49	return nil
	50	}
	51
	52	// Bytes is equivalent to bytes.Buffer.Bytes.
	53	func (buf *Buffer) Bytes() []byte {
	54	return buf.bytes
	55	}
	56
	57	// Strings is equivalent to bytes.Buffer.Strings.
	58	func (buf *Buffer) String() string {
	59	return string(buf.bytes)
	60	}
	61
	62	// Len is equivalent to bytes.Buffer.Len.
	63	func (buf *Buffer) Len() int {
	64	return len(buf.bytes)
	65	}

vendor/github.com/xwb1989/sqlparser/dependency/hack/hack.go+79−0

@@ -0,0 +1,79 @@
	1	/*
	2	Copyright 2017 Google Inc.
	3
	4	Licensed under the Apache License, Version 2.0 (the "License");
	5	you may not use this file except in compliance with the License.
	6	You may obtain a copy of the License at
	7
	8	http://www.apache.org/licenses/LICENSE-2.0
	9
	10	Unless required by applicable law or agreed to in writing, software
	11	distributed under the License is distributed on an "AS IS" BASIS,
	12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	See the License for the specific language governing permissions and
	14	limitations under the License.
	15	*/
	16
	17	// Package hack gives you some efficient functionality at the cost of
	18	// breaking some Go rules.
	19	package hack
	20
	21	import (
	22	"reflect"
	23	"unsafe"
	24	)
	25
	26	// StringArena lets you consolidate allocations for a group of strings
	27	// that have similar life length
	28	type StringArena struct {
	29	buf []byte
	30	str string
	31	}
	32
	33	// NewStringArena creates an arena of the specified size.
	34	func NewStringArena(size int) *StringArena {
	35	sa := &StringArena{buf: make([]byte, 0, size)}
	36	pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&sa.buf))
	37	pstring := (*reflect.StringHeader)(unsafe.Pointer(&sa.str))
	38	pstring.Data = pbytes.Data
	39	pstring.Len = pbytes.Cap
	40	return sa
	41	}
	42
	43	// NewString copies a byte slice into the arena and returns it as a string.
	44	// If the arena is full, it returns a traditional go string.
	45	func (sa *StringArena) NewString(b []byte) string {
	46	if len(b) == 0 {
	47	return ""
	48	}
	49	if len(sa.buf)+len(b) > cap(sa.buf) {
	50	return string(b)
	51	}
	52	start := len(sa.buf)
	53	sa.buf = append(sa.buf, b...)
	54	return sa.str[start : start+len(b)]
	55	}
	56
	57	// SpaceLeft returns the amount of space left in the arena.
	58	func (sa *StringArena) SpaceLeft() int {
	59	return cap(sa.buf) - len(sa.buf)
	60	}
	61
	62	// String force casts a []byte to a string.
	63	// USE AT YOUR OWN RISK
	64	func String(b []byte) (s string) {
	65	if len(b) == 0 {
	66	return ""
	67	}
	68	pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&b))
	69	pstring := (*reflect.StringHeader)(unsafe.Pointer(&s))
	70	pstring.Data = pbytes.Data
	71	pstring.Len = pbytes.Len
	72	return
	73	}
	74
	75	// StringPointer returns &s[0], which is not allowed in go
	76	func StringPointer(s string) unsafe.Pointer {
	77	pstring := (*reflect.StringHeader)(unsafe.Pointer(&s))
	78	return unsafe.Pointer(pstring.Data)
	79	}

vendor/github.com/xwb1989/sqlparser/dependency/querypb/query.pb.go+2734−0

@@ -0,0 +1,2734 @@
	1	// Code generated by protoc-gen-go. DO NOT EDIT.
	2	// source: query.proto
	3
	4	/*
	5	Package query is a generated protocol buffer package.
	6
	7	It is generated from these files:
	8	query.proto
	9
	10	It has these top-level messages:
	11	Target
	12	VTGateCallerID
	13	EventToken
	14	Value
	15	BindVariable
	16	BoundQuery
	17	ExecuteOptions
	18	Field
	19	Row
	20	ResultExtras
	21	QueryResult
	22	StreamEvent
	23	ExecuteRequest
	24	ExecuteResponse
	25	ResultWithError
	26	ExecuteBatchRequest
	27	ExecuteBatchResponse
	28	StreamExecuteRequest
	29	StreamExecuteResponse
	30	BeginRequest
	31	BeginResponse
	32	CommitRequest
	33	CommitResponse
	34	RollbackRequest
	35	RollbackResponse
	36	PrepareRequest
	37	PrepareResponse
	38	CommitPreparedRequest
	39	CommitPreparedResponse
	40	RollbackPreparedRequest
	41	RollbackPreparedResponse
	42	CreateTransactionRequest
	43	CreateTransactionResponse
	44	StartCommitRequest
	45	StartCommitResponse
	46	SetRollbackRequest
	47	SetRollbackResponse
	48	ConcludeTransactionRequest
	49	ConcludeTransactionResponse
	50	ReadTransactionRequest
	51	ReadTransactionResponse
	52	BeginExecuteRequest
	53	BeginExecuteResponse
	54	BeginExecuteBatchRequest
	55	BeginExecuteBatchResponse
	56	MessageStreamRequest
	57	MessageStreamResponse
	58	MessageAckRequest
	59	MessageAckResponse
	60	SplitQueryRequest
	61	QuerySplit
	62	SplitQueryResponse
	63	StreamHealthRequest
	64	RealtimeStats
	65	StreamHealthResponse
	66	UpdateStreamRequest
	67	UpdateStreamResponse
	68	TransactionMetadata
	69	*/
	70	package querypb
	71
	72	import "strconv"
	73
	74	// EnumName is a helper function to simplify printing protocol buffer enums
	75	// by name. Given an enum map and a value, it returns a useful string.
	76	func EnumName(m map[int32]string, v int32) string {
	77	s, ok := m[v]
	78	if ok {
	79	return s
	80	}
	81	return strconv.Itoa(int(v))
	82	}
	83
	84	// Flags sent from the MySQL C API
	85	type MySqlFlag int32
	86
	87	const (
	88	MySqlFlag_EMPTY MySqlFlag = 0
	89	MySqlFlag_NOT_NULL_FLAG MySqlFlag = 1
	90	MySqlFlag_PRI_KEY_FLAG MySqlFlag = 2
	91	MySqlFlag_UNIQUE_KEY_FLAG MySqlFlag = 4
	92	MySqlFlag_MULTIPLE_KEY_FLAG MySqlFlag = 8
	93	MySqlFlag_BLOB_FLAG MySqlFlag = 16
	94	MySqlFlag_UNSIGNED_FLAG MySqlFlag = 32
	95	MySqlFlag_ZEROFILL_FLAG MySqlFlag = 64
	96	MySqlFlag_BINARY_FLAG MySqlFlag = 128
	97	MySqlFlag_ENUM_FLAG MySqlFlag = 256
	98	MySqlFlag_AUTO_INCREMENT_FLAG MySqlFlag = 512
	99	MySqlFlag_TIMESTAMP_FLAG MySqlFlag = 1024
	100	MySqlFlag_SET_FLAG MySqlFlag = 2048
	101	MySqlFlag_NO_DEFAULT_VALUE_FLAG MySqlFlag = 4096
	102	MySqlFlag_ON_UPDATE_NOW_FLAG MySqlFlag = 8192
	103	MySqlFlag_NUM_FLAG MySqlFlag = 32768
	104	MySqlFlag_PART_KEY_FLAG MySqlFlag = 16384
	105	MySqlFlag_GROUP_FLAG MySqlFlag = 32768
	106	MySqlFlag_UNIQUE_FLAG MySqlFlag = 65536
	107	MySqlFlag_BINCMP_FLAG MySqlFlag = 131072
	108	)
	109
	110	var MySqlFlag_name = map[int32]string{
	111	0: "EMPTY",
	112	1: "NOT_NULL_FLAG",
	113	2: "PRI_KEY_FLAG",
	114	4: "UNIQUE_KEY_FLAG",
	115	8: "MULTIPLE_KEY_FLAG",
	116	16: "BLOB_FLAG",
	117	32: "UNSIGNED_FLAG",
	118	64: "ZEROFILL_FLAG",
	119	128: "BINARY_FLAG",
	120	256: "ENUM_FLAG",
	121	512: "AUTO_INCREMENT_FLAG",
	122	1024: "TIMESTAMP_FLAG",
	123	2048: "SET_FLAG",
	124	4096: "NO_DEFAULT_VALUE_FLAG",
	125	8192: "ON_UPDATE_NOW_FLAG",
	126	32768: "NUM_FLAG",
	127	16384: "PART_KEY_FLAG",
	128	// Duplicate value: 32768: "GROUP_FLAG",
	129	65536: "UNIQUE_FLAG",
	130	131072: "BINCMP_FLAG",
	131	}
	132	var MySqlFlag_value = map[string]int32{
	133	"EMPTY": 0,
	134	"NOT_NULL_FLAG": 1,
	135	"PRI_KEY_FLAG": 2,
	136	"UNIQUE_KEY_FLAG": 4,
	137	"MULTIPLE_KEY_FLAG": 8,
	138	"BLOB_FLAG": 16,
	139	"UNSIGNED_FLAG": 32,
	140	"ZEROFILL_FLAG": 64,
	141	"BINARY_FLAG": 128,
	142	"ENUM_FLAG": 256,
	143	"AUTO_INCREMENT_FLAG": 512,
	144	"TIMESTAMP_FLAG": 1024,
	145	"SET_FLAG": 2048,
	146	"NO_DEFAULT_VALUE_FLAG": 4096,
	147	"ON_UPDATE_NOW_FLAG": 8192,
	148	"NUM_FLAG": 32768,
	149	"PART_KEY_FLAG": 16384,
	150	"GROUP_FLAG": 32768,
	151	"UNIQUE_FLAG": 65536,
	152	"BINCMP_FLAG": 131072,
	153	}
	154
	155	func (x MySqlFlag) String() string {
	156	return EnumName(MySqlFlag_name, int32(x))
	157	}
	158
	159	// Flag allows us to qualify types by their common properties.
	160	type Flag int32
	161
	162	const (
	163	Flag_NONE Flag = 0
	164	Flag_ISINTEGRAL Flag = 256
	165	Flag_ISUNSIGNED Flag = 512
	166	Flag_ISFLOAT Flag = 1024
	167	Flag_ISQUOTED Flag = 2048
	168	Flag_ISTEXT Flag = 4096
	169	Flag_ISBINARY Flag = 8192
	170	)
	171
	172	var Flag_name = map[int32]string{
	173	0: "NONE",
	174	256: "ISINTEGRAL",
	175	512: "ISUNSIGNED",
	176	1024: "ISFLOAT",
	177	2048: "ISQUOTED",
	178	4096: "ISTEXT",
	179	8192: "ISBINARY",
	180	}
	181	var Flag_value = map[string]int32{
	182	"NONE": 0,
	183	"ISINTEGRAL": 256,
	184	"ISUNSIGNED": 512,
	185	"ISFLOAT": 1024,
	186	"ISQUOTED": 2048,
	187	"ISTEXT": 4096,
	188	"ISBINARY": 8192,
	189	}
	190
	191	func (x Flag) String() string {
	192	return EnumName(Flag_name, int32(x))
	193	}
	194
	195	// Type defines the various supported data types in bind vars
	196	// and query results.
	197	type Type int32
	198
	199	const (
	200	// NULL_TYPE specifies a NULL type.
	201	Type_NULL_TYPE Type = 0
	202	// INT8 specifies a TINYINT type.
	203	// Properties: 1, IsNumber.
	204	Type_INT8 Type = 257
	205	// UINT8 specifies a TINYINT UNSIGNED type.
	206	// Properties: 2, IsNumber, IsUnsigned.
	207	Type_UINT8 Type = 770
	208	// INT16 specifies a SMALLINT type.
	209	// Properties: 3, IsNumber.
	210	Type_INT16 Type = 259
	211	// UINT16 specifies a SMALLINT UNSIGNED type.
	212	// Properties: 4, IsNumber, IsUnsigned.
	213	Type_UINT16 Type = 772
	214	// INT24 specifies a MEDIUMINT type.
	215	// Properties: 5, IsNumber.
	216	Type_INT24 Type = 261
	217	// UINT24 specifies a MEDIUMINT UNSIGNED type.
	218	// Properties: 6, IsNumber, IsUnsigned.
	219	Type_UINT24 Type = 774
	220	// INT32 specifies a INTEGER type.
	221	// Properties: 7, IsNumber.
	222	Type_INT32 Type = 263
	223	// UINT32 specifies a INTEGER UNSIGNED type.
	224	// Properties: 8, IsNumber, IsUnsigned.
	225	Type_UINT32 Type = 776
	226	// INT64 specifies a BIGINT type.
	227	// Properties: 9, IsNumber.
	228	Type_INT64 Type = 265
	229	// UINT64 specifies a BIGINT UNSIGNED type.
	230	// Properties: 10, IsNumber, IsUnsigned.
	231	Type_UINT64 Type = 778
	232	// FLOAT32 specifies a FLOAT type.
	233	// Properties: 11, IsFloat.
	234	Type_FLOAT32 Type = 1035
	235	// FLOAT64 specifies a DOUBLE or REAL type.
	236	// Properties: 12, IsFloat.
	237	Type_FLOAT64 Type = 1036
	238	// TIMESTAMP specifies a TIMESTAMP type.
	239	// Properties: 13, IsQuoted.
	240	Type_TIMESTAMP Type = 2061
	241	// DATE specifies a DATE type.
	242	// Properties: 14, IsQuoted.
	243	Type_DATE Type = 2062
	244	// TIME specifies a TIME type.
	245	// Properties: 15, IsQuoted.
	246	Type_TIME Type = 2063
	247	// DATETIME specifies a DATETIME type.
	248	// Properties: 16, IsQuoted.
	249	Type_DATETIME Type = 2064
	250	// YEAR specifies a YEAR type.
	251	// Properties: 17, IsNumber, IsUnsigned.
	252	Type_YEAR Type = 785
	253	// DECIMAL specifies a DECIMAL or NUMERIC type.
	254	// Properties: 18, None.
	255	Type_DECIMAL Type = 18
	256	// TEXT specifies a TEXT type.
	257	// Properties: 19, IsQuoted, IsText.
	258	Type_TEXT Type = 6163
	259	// BLOB specifies a BLOB type.
	260	// Properties: 20, IsQuoted, IsBinary.
	261	Type_BLOB Type = 10260
	262	// VARCHAR specifies a VARCHAR type.
	263	// Properties: 21, IsQuoted, IsText.
	264	Type_VARCHAR Type = 6165
	265	// VARBINARY specifies a VARBINARY type.
	266	// Properties: 22, IsQuoted, IsBinary.
	267	Type_VARBINARY Type = 10262
	268	// CHAR specifies a CHAR type.
	269	// Properties: 23, IsQuoted, IsText.
	270	Type_CHAR Type = 6167
	271	// BINARY specifies a BINARY type.
	272	// Properties: 24, IsQuoted, IsBinary.
	273	Type_BINARY Type = 10264
	274	// BIT specifies a BIT type.
	275	// Properties: 25, IsQuoted.
	276	Type_BIT Type = 2073
	277	// ENUM specifies an ENUM type.
	278	// Properties: 26, IsQuoted.
	279	Type_ENUM Type = 2074
	280	// SET specifies a SET type.
	281	// Properties: 27, IsQuoted.
	282	Type_SET Type = 2075
	283	// TUPLE specifies a a tuple. This cannot
	284	// be returned in a QueryResult, but it can
	285	// be sent as a bind var.
	286	// Properties: 28, None.
	287	Type_TUPLE Type = 28
	288	// GEOMETRY specifies a GEOMETRY type.
	289	// Properties: 29, IsQuoted.
	290	Type_GEOMETRY Type = 2077
	291	// JSON specifies a JSON type.
	292	// Properties: 30, IsQuoted.
	293	Type_JSON Type = 2078
	294	// EXPRESSION specifies a SQL expression.
	295	// This type is for internal use only.
	296	// Properties: 31, None.
	297	Type_EXPRESSION Type = 31
	298	)
	299
	300	var Type_name = map[int32]string{
	301	0: "NULL_TYPE",
	302	257: "INT8",
	303	770: "UINT8",
	304	259: "INT16",
	305	772: "UINT16",
	306	261: "INT24",
	307	774: "UINT24",
	308	263: "INT32",
	309	776: "UINT32",
	310	265: "INT64",
	311	778: "UINT64",
	312	1035: "FLOAT32",
	313	1036: "FLOAT64",
	314	2061: "TIMESTAMP",
	315	2062: "DATE",
	316	2063: "TIME",
	317	2064: "DATETIME",
	318	785: "YEAR",
	319	18: "DECIMAL",
	320	6163: "TEXT",
	321	10260: "BLOB",
	322	6165: "VARCHAR",
	323	10262: "VARBINARY",
	324	6167: "CHAR",
	325	10264: "BINARY",
	326	2073: "BIT",
	327	2074: "ENUM",
	328	2075: "SET",
	329	28: "TUPLE",
	330	2077: "GEOMETRY",
	331	2078: "JSON",
	332	31: "EXPRESSION",
	333	}
	334	var Type_value = map[string]int32{
	335	"NULL_TYPE": 0,
	336	"INT8": 257,
	337	"UINT8": 770,
	338	"INT16": 259,
	339	"UINT16": 772,
	340	"INT24": 261,
	341	"UINT24": 774,
	342	"INT32": 263,
	343	"UINT32": 776,
	344	"INT64": 265,
	345	"UINT64": 778,
	346	"FLOAT32": 1035,
	347	"FLOAT64": 1036,
	348	"TIMESTAMP": 2061,
	349	"DATE": 2062,
	350	"TIME": 2063,
	351	"DATETIME": 2064,
	352	"YEAR": 785,
	353	"DECIMAL": 18,
	354	"TEXT": 6163,
	355	"BLOB": 10260,
	356	"VARCHAR": 6165,
	357	"VARBINARY": 10262,
	358	"CHAR": 6167,
	359	"BINARY": 10264,
	360	"BIT": 2073,
	361	"ENUM": 2074,
	362	"SET": 2075,
	363	"TUPLE": 28,
	364	"GEOMETRY": 2077,
	365	"JSON": 2078,
	366	"EXPRESSION": 31,
	367	}
	368
	369	func (x Type) String() string {
	370	return EnumName(Type_name, int32(x))
	371	}
	372
	373	// TransactionState represents the state of a distributed transaction.
	374	type TransactionState int32
	375
	376	const (
	377	TransactionState_UNKNOWN TransactionState = 0
	378	TransactionState_PREPARE TransactionState = 1
	379	TransactionState_COMMIT TransactionState = 2
	380	TransactionState_ROLLBACK TransactionState = 3
	381	)
	382
	383	var TransactionState_name = map[int32]string{
	384	0: "UNKNOWN",
	385	1: "PREPARE",
	386	2: "COMMIT",
	387	3: "ROLLBACK",
	388	}
	389	var TransactionState_value = map[string]int32{
	390	"UNKNOWN": 0,
	391	"PREPARE": 1,
	392	"COMMIT": 2,
	393	"ROLLBACK": 3,
	394	}
	395
	396	func (x TransactionState) String() string {
	397	return EnumName(TransactionState_name, int32(x))
	398	}
	399
	400	type ExecuteOptions_IncludedFields int32
	401
	402	const (
	403	ExecuteOptions_TYPE_AND_NAME ExecuteOptions_IncludedFields = 0
	404	ExecuteOptions_TYPE_ONLY ExecuteOptions_IncludedFields = 1
	405	ExecuteOptions_ALL ExecuteOptions_IncludedFields = 2
	406	)
	407
	408	var ExecuteOptions_IncludedFields_name = map[int32]string{
	409	0: "TYPE_AND_NAME",
	410	1: "TYPE_ONLY",
	411	2: "ALL",
	412	}
	413	var ExecuteOptions_IncludedFields_value = map[string]int32{
	414	"TYPE_AND_NAME": 0,
	415	"TYPE_ONLY": 1,
	416	"ALL": 2,
	417	}
	418
	419	func (x ExecuteOptions_IncludedFields) String() string {
	420	return EnumName(ExecuteOptions_IncludedFields_name, int32(x))
	421	}
	422
	423	type ExecuteOptions_Workload int32
	424
	425	const (
	426	ExecuteOptions_UNSPECIFIED ExecuteOptions_Workload = 0
	427	ExecuteOptions_OLTP ExecuteOptions_Workload = 1
	428	ExecuteOptions_OLAP ExecuteOptions_Workload = 2
	429	ExecuteOptions_DBA ExecuteOptions_Workload = 3
	430	)
	431
	432	var ExecuteOptions_Workload_name = map[int32]string{
	433	0: "UNSPECIFIED",
	434	1: "OLTP",
	435	2: "OLAP",
	436	3: "DBA",
	437	}
	438	var ExecuteOptions_Workload_value = map[string]int32{
	439	"UNSPECIFIED": 0,
	440	"OLTP": 1,
	441	"OLAP": 2,
	442	"DBA": 3,
	443	}
	444
	445	func (x ExecuteOptions_Workload) String() string {
	446	return EnumName(ExecuteOptions_Workload_name, int32(x))
	447	}
	448
	449	type ExecuteOptions_TransactionIsolation int32
	450
	451	const (
	452	ExecuteOptions_DEFAULT ExecuteOptions_TransactionIsolation = 0
	453	ExecuteOptions_REPEATABLE_READ ExecuteOptions_TransactionIsolation = 1
	454	ExecuteOptions_READ_COMMITTED ExecuteOptions_TransactionIsolation = 2
	455	ExecuteOptions_READ_UNCOMMITTED ExecuteOptions_TransactionIsolation = 3
	456	ExecuteOptions_SERIALIZABLE ExecuteOptions_TransactionIsolation = 4
	457	)
	458
	459	var ExecuteOptions_TransactionIsolation_name = map[int32]string{
	460	0: "DEFAULT",
	461	1: "REPEATABLE_READ",
	462	2: "READ_COMMITTED",
	463	3: "READ_UNCOMMITTED",
	464	4: "SERIALIZABLE",
	465	}
	466	var ExecuteOptions_TransactionIsolation_value = map[string]int32{
	467	"DEFAULT": 0,
	468	"REPEATABLE_READ": 1,
	469	"READ_COMMITTED": 2,
	470	"READ_UNCOMMITTED": 3,
	471	"SERIALIZABLE": 4,
	472	}
	473
	474	func (x ExecuteOptions_TransactionIsolation) String() string {
	475	return EnumName(ExecuteOptions_TransactionIsolation_name, int32(x))
	476	}
	477
	478	// The category of one statement.
	479	type StreamEvent_Statement_Category int32
	480
	481	const (
	482	StreamEvent_Statement_Error StreamEvent_Statement_Category = 0
	483	StreamEvent_Statement_DML StreamEvent_Statement_Category = 1
	484	StreamEvent_Statement_DDL StreamEvent_Statement_Category = 2
	485	)
	486
	487	var StreamEvent_Statement_Category_name = map[int32]string{
	488	0: "Error",
	489	1: "DML",
	490	2: "DDL",
	491	}
	492	var StreamEvent_Statement_Category_value = map[string]int32{
	493	"Error": 0,
	494	"DML": 1,
	495	"DDL": 2,
	496	}
	497
	498	func (x StreamEvent_Statement_Category) String() string {
	499	return EnumName(StreamEvent_Statement_Category_name, int32(x))
	500	}
	501
	502	type SplitQueryRequest_Algorithm int32
	503
	504	const (
	505	SplitQueryRequest_EQUAL_SPLITS SplitQueryRequest_Algorithm = 0
	506	SplitQueryRequest_FULL_SCAN SplitQueryRequest_Algorithm = 1
	507	)
	508
	509	var SplitQueryRequest_Algorithm_name = map[int32]string{
	510	0: "EQUAL_SPLITS",
	511	1: "FULL_SCAN",
	512	}
	513	var SplitQueryRequest_Algorithm_value = map[string]int32{
	514	"EQUAL_SPLITS": 0,
	515	"FULL_SCAN": 1,
	516	}
	517
	518	func (x SplitQueryRequest_Algorithm) String() string {
	519	return EnumName(SplitQueryRequest_Algorithm_name, int32(x))
	520	}
	521
	522	/*
	523	// Target describes what the client expects the tablet is.
	524	// If the tablet does not match, an error is returned.
	525	type Target struct {
	526	Keyspace string `protobuf:"bytes,1,opt,name=keyspace" json:"keyspace,omitempty"`
	527	Shard string `protobuf:"bytes,2,opt,name=shard" json:"shard,omitempty"`
	528	TabletType topodata.TabletType `protobuf:"varint,3,opt,name=tablet_type,json=tabletType,enum=topodata.TabletType" json:"tablet_type,omitempty"`
	529	}
	530
	531	func (m Target) Reset() { m = Target{} }
	532	func (m *Target) String() string { return "TODO" }
	533
	534
	535
	536	func (m *Target) GetKeyspace() string {
	537	if m != nil {
	538	return m.Keyspace
	539	}
	540	return ""
	541	}
	542
	543	func (m *Target) GetShard() string {
	544	if m != nil {
	545	return m.Shard
	546	}
	547	return ""
	548	}
	549
	550	func (m *Target) GetTabletType() topodata.TabletType {
	551	if m != nil {
	552	return m.TabletType
	553	}
	554	return topodata.TabletType_UNKNOWN
	555	}
	556
	557
	558	// VTGateCallerID is sent by VTGate to VTTablet to describe the
	559	// caller. If possible, this information is secure. For instance,
	560	// if using unique certificates that guarantee that VTGate->VTTablet
	561	// traffic cannot be spoofed, then VTTablet can trust this information,
	562	// and VTTablet will use it for tablet ACLs, for instance.
	563	// Because of this security guarantee, this is different than the CallerID
	564	// structure, which is not secure at all, because it is provided
	565	// by the Vitess client.
	566	type VTGateCallerID struct {
	567	Username string `protobuf:"bytes,1,opt,name=username" json:"username,omitempty"`
	568	Groups []string `protobuf:"bytes,2,rep,name=groups" json:"groups,omitempty"`
	569	}
	570
	571	func (m VTGateCallerID) Reset() { m = VTGateCallerID{} }
	572	func (m *VTGateCallerID) String() string { return "TODO" }
	573
	574
	575
	576	func (m *VTGateCallerID) GetUsername() string {
	577	if m != nil {
	578	return m.Username
	579	}
	580	return ""
	581	}
	582
	583	func (m *VTGateCallerID) GetGroups() []string {
	584	if m != nil {
	585	return m.Groups
	586	}
	587	return nil
	588	}
	589
	590	// EventToken is a structure that describes a point in time in a
	591	// replication stream on one shard. The most recent known replication
	592	// position can be retrieved from vttablet when executing a query. It
	593	// is also sent with the replication streams from the binlog service.
	594	type EventToken struct {
	595	// timestamp is the MySQL timestamp of the statements. Seconds since Epoch.
	596	Timestamp int64 `protobuf:"varint,1,opt,name=timestamp" json:"timestamp,omitempty"`
	597	// The shard name that applied the statements. Note this is not set when
	598	// streaming from a vttablet. It is only used on the client -> vtgate link.
	599	Shard string `protobuf:"bytes,2,opt,name=shard" json:"shard,omitempty"`
	600	// The position on the replication stream after this statement was applied.
	601	// It is not the transaction ID / GTID, but the position / GTIDSet.
	602	Position string `protobuf:"bytes,3,opt,name=position" json:"position,omitempty"`
	603	}
	604
	605	func (m EventToken) Reset() { m = EventToken{} }
	606	func (m *EventToken) String() string { return "TODO" }
	607
	608
	609
	610	func (m *EventToken) GetTimestamp() int64 {
	611	if m != nil {
	612	return m.Timestamp
	613	}
	614	return 0
	615	}
	616
	617	func (m *EventToken) GetShard() string {
	618	if m != nil {
	619	return m.Shard
	620	}
	621	return ""
	622	}
	623
	624	func (m *EventToken) GetPosition() string {
	625	if m != nil {
	626	return m.Position
	627	}
	628	return ""
	629	}
	630	*/
	631
	632	// Value represents a typed value.
	633	type Value struct {
	634	Type Type `protobuf:"varint,1,opt,name=type,enum=query.Type" json:"type,omitempty"`
	635	Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
	636	}
	637
	638	func (m Value) Reset() { m = Value{} }
	639	func (m *Value) String() string { return "TODO" }
	640
	641	func (m *Value) GetType() Type {
	642	if m != nil {
	643	return m.Type
	644	}
	645	return Type_NULL_TYPE
	646	}
	647
	648	func (m *Value) GetValue() []byte {
	649	if m != nil {
	650	return m.Value
	651	}
	652	return nil
	653	}
	654
	655	// BindVariable represents a single bind variable in a Query.
	656	type BindVariable struct {
	657	Type Type `protobuf:"varint,1,opt,name=type,enum=query.Type" json:"type,omitempty"`
	658	Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
	659	// values are set if type is TUPLE.
	660	Values []*Value `protobuf:"bytes,3,rep,name=values" json:"values,omitempty"`
	661	}
	662
	663	func (m BindVariable) Reset() { m = BindVariable{} }
	664	func (m *BindVariable) String() string { return "TODO" }
	665
	666	func (m *BindVariable) GetType() Type {
	667	if m != nil {
	668	return m.Type
	669	}
	670	return Type_NULL_TYPE
	671	}
	672
	673	func (m *BindVariable) GetValue() []byte {
	674	if m != nil {
	675	return m.Value
	676	}
	677	return nil
	678	}
	679
	680	func (m BindVariable) GetValues() []Value {
	681	if m != nil {
	682	return m.Values
	683	}
	684	return nil
	685	}
	686
	687	// BoundQuery is a query with its bind variables
	688	type BoundQuery struct {
	689	// sql is the SQL query to execute
	690	Sql string `protobuf:"bytes,1,opt,name=sql" json:"sql,omitempty"`
	691	// bind_variables is a map of all bind variables to expand in the query.
	692	// nil values are not allowed. Use NULL_TYPE to express a NULL value.
	693	BindVariables map[string]*BindVariable `protobuf:"bytes,2,rep,name=bind_variables,json=bindVariables" json:"bind_variables,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
	694	}
	695
	696	func (m BoundQuery) Reset() { m = BoundQuery{} }
	697	func (m *BoundQuery) String() string { return "TODO" }
	698
	699	func (m *BoundQuery) GetSql() string {
	700	if m != nil {
	701	return m.Sql
	702	}
	703	return ""
	704	}
	705
	706	func (m BoundQuery) GetBindVariables() map[string]BindVariable {
	707	if m != nil {
	708	return m.BindVariables
	709	}
	710	return nil
	711	}
	712
	713	/*
	714	// ExecuteOptions is passed around for all Execute calls.
	715	type ExecuteOptions struct {
	716	// If set, we will try to include an EventToken with the responses.
	717	IncludeEventToken bool `protobuf:"varint,2,opt,name=include_event_token,json=includeEventToken" json:"include_event_token,omitempty"`
	718	// If set, the fresher field may be set as a result comparison to this token.
	719	// This is a shortcut so the application doesn't need to care about
	720	// comparing EventTokens.
	721	CompareEventToken *EventToken `protobuf:"bytes,3,opt,name=compare_event_token,json=compareEventToken" json:"compare_event_token,omitempty"`
	722	// Controls what fields are returned in Field message responses from mysql, i.e.
	723	// field name, table name, etc. This is an optimization for high-QPS queries where
	724	// the client knows what it's getting
	725	IncludedFields ExecuteOptions_IncludedFields `protobuf:"varint,4,opt,name=included_fields,json=includedFields,enum=query.ExecuteOptions_IncludedFields" json:"included_fields,omitempty"`
	726	// client_rows_found specifies if rows_affected should return
	727	// rows found instead of rows affected. Behavior is defined
	728	// by MySQL's CLIENT_FOUND_ROWS flag.
	729	ClientFoundRows bool `protobuf:"varint,5,opt,name=client_found_rows,json=clientFoundRows" json:"client_found_rows,omitempty"`
	730	// workload specifies the type of workload:
	731	// OLTP: DMLs allowed, results have row count limit, and
	732	// query timeouts are shorter.
	733	// OLAP: DMLS not allowed, no limit on row count, timeouts
	734	// can be as high as desired.
	735	// DBA: no limit on rowcount or timeout, all queries allowed
	736	// but intended for long DMLs and DDLs.
	737	Workload ExecuteOptions_Workload `protobuf:"varint,6,opt,name=workload,enum=query.ExecuteOptions_Workload" json:"workload,omitempty"`
	738	// sql_select_limit sets an implicit limit on all select statements. Since
	739	// vitess also sets a rowcount limit on queries, the smallest value wins.
	740	SqlSelectLimit int64 `protobuf:"varint,8,opt,name=sql_select_limit,json=sqlSelectLimit" json:"sql_select_limit,omitempty"`
	741	TransactionIsolation ExecuteOptions_TransactionIsolation `protobuf:"varint,9,opt,name=transaction_isolation,json=transactionIsolation,enum=query.ExecuteOptions_TransactionIsolation" json:"transaction_isolation,omitempty"`
	742	}
	743
	744	func (m ExecuteOptions) Reset() { m = ExecuteOptions{} }
	745	func (m *ExecuteOptions) String() string { return "TODO" }
	746
	747
	748
	749	func (m *ExecuteOptions) GetIncludeEventToken() bool {
	750	if m != nil {
	751	return m.IncludeEventToken
	752	}
	753	return false
	754	}
	755
	756	func (m ExecuteOptions) GetCompareEventToken() EventToken {
	757	if m != nil {
	758	return m.CompareEventToken
	759	}
	760	return nil
	761	}
	762
	763	func (m *ExecuteOptions) GetIncludedFields() ExecuteOptions_IncludedFields {
	764	if m != nil {
	765	return m.IncludedFields
	766	}
	767	return ExecuteOptions_TYPE_AND_NAME
	768	}
	769
	770	func (m *ExecuteOptions) GetClientFoundRows() bool {
	771	if m != nil {
	772	return m.ClientFoundRows
	773	}
	774	return false
	775	}
	776
	777	func (m *ExecuteOptions) GetWorkload() ExecuteOptions_Workload {
	778	if m != nil {
	779	return m.Workload
	780	}
	781	return ExecuteOptions_UNSPECIFIED
	782	}
	783
	784	func (m *ExecuteOptions) GetSqlSelectLimit() int64 {
	785	if m != nil {
	786	return m.SqlSelectLimit
	787	}
	788	return 0
	789	}
	790
	791	func (m *ExecuteOptions) GetTransactionIsolation() ExecuteOptions_TransactionIsolation {
	792	if m != nil {
	793	return m.TransactionIsolation
	794	}
	795	return ExecuteOptions_DEFAULT
	796	}
	797
	798	// Field describes a single column returned by a query
	799	type Field struct {
	800	// name of the field as returned by mysql C API
	801	Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
	802	// vitess-defined type. Conversion function is in sqltypes package.
	803	Type Type `protobuf:"varint,2,opt,name=type,enum=query.Type" json:"type,omitempty"`
	804	// Remaining fields from mysql C API.
	805	// These fields are only populated when ExecuteOptions.included_fields
	806	// is set to IncludedFields.ALL.
	807	Table string `protobuf:"bytes,3,opt,name=table" json:"table,omitempty"`
	808	OrgTable string `protobuf:"bytes,4,opt,name=org_table,json=orgTable" json:"org_table,omitempty"`
	809	Database string `protobuf:"bytes,5,opt,name=database" json:"database,omitempty"`
	810	OrgName string `protobuf:"bytes,6,opt,name=org_name,json=orgName" json:"org_name,omitempty"`
	811	// column_length is really a uint32. All 32 bits can be used.
	812	ColumnLength uint32 `protobuf:"varint,7,opt,name=column_length,json=columnLength" json:"column_length,omitempty"`
	813	// charset is actually a uint16. Only the lower 16 bits are used.
	814	Charset uint32 `protobuf:"varint,8,opt,name=charset" json:"charset,omitempty"`
	815	// decimals is actualy a uint8. Only the lower 8 bits are used.
	816	Decimals uint32 `protobuf:"varint,9,opt,name=decimals" json:"decimals,omitempty"`
	817	// flags is actually a uint16. Only the lower 16 bits are used.
	818	Flags uint32 `protobuf:"varint,10,opt,name=flags" json:"flags,omitempty"`
	819	}
	820
	821	func (m Field) Reset() { m = Field{} }
	822	func (m *Field) String() string { return "TODO" }
	823
	824
	825
	826	func (m *Field) GetName() string {
	827	if m != nil {
	828	return m.Name
	829	}
	830	return ""
	831	}
	832
	833	func (m *Field) GetType() Type {
	834	if m != nil {
	835	return m.Type
	836	}
	837	return Type_NULL_TYPE
	838	}
	839
	840	func (m *Field) GetTable() string {
	841	if m != nil {
	842	return m.Table
	843	}
	844	return ""
	845	}
	846
	847	func (m *Field) GetOrgTable() string {
	848	if m != nil {
	849	return m.OrgTable
	850	}
	851	return ""
	852	}
	853
	854	func (m *Field) GetDatabase() string {
	855	if m != nil {
	856	return m.Database
	857	}
	858	return ""
	859	}
	860
	861	func (m *Field) GetOrgName() string {
	862	if m != nil {
	863	return m.OrgName
	864	}
	865	return ""
	866	}
	867
	868	func (m *Field) GetColumnLength() uint32 {
	869	if m != nil {
	870	return m.ColumnLength
	871	}
	872	return 0
	873	}
	874
	875	func (m *Field) GetCharset() uint32 {
	876	if m != nil {
	877	return m.Charset
	878	}
	879	return 0
	880	}
	881
	882	func (m *Field) GetDecimals() uint32 {
	883	if m != nil {
	884	return m.Decimals
	885	}
	886	return 0
	887	}
	888
	889	func (m *Field) GetFlags() uint32 {
	890	if m != nil {
	891	return m.Flags
	892	}
	893	return 0
	894	}
	895
	896	// Row is a database row.
	897	type Row struct {
	898	// lengths contains the length of each value in values.
	899	// A length of -1 means that the field is NULL. While
	900	// reading values, you have to accummulate the length
	901	// to know the offset where the next value begins in values.
	902	Lengths []int64 `protobuf:"zigzag64,1,rep,packed,name=lengths" json:"lengths,omitempty"`
	903	// values contains a concatenation of all values in the row.
	904	Values []byte `protobuf:"bytes,2,opt,name=values,proto3" json:"values,omitempty"`
	905	}
	906
	907	func (m Row) Reset() { m = Row{} }
	908	func (m *Row) String() string { return "TODO" }
	909
	910
	911
	912	func (m *Row) GetLengths() []int64 {
	913	if m != nil {
	914	return m.Lengths
	915	}
	916	return nil
	917	}
	918
	919	func (m *Row) GetValues() []byte {
	920	if m != nil {
	921	return m.Values
	922	}
	923	return nil
	924	}
	925
	926	// ResultExtras contains optional out-of-band information. Usually the
	927	// extras are requested by adding ExecuteOptions flags.
	928	type ResultExtras struct {
	929	// event_token is populated if the include_event_token flag is set
	930	// in ExecuteOptions.
	931	EventToken *EventToken `protobuf:"bytes,1,opt,name=event_token,json=eventToken" json:"event_token,omitempty"`
	932	// If set, it means the data returned with this result is fresher
	933	// than the compare_token passed in the ExecuteOptions.
	934	Fresher bool `protobuf:"varint,2,opt,name=fresher" json:"fresher,omitempty"`
	935	}
	936
	937	func (m ResultExtras) Reset() { m = ResultExtras{} }
	938	func (m *ResultExtras) String() string { return "TODO" }
	939
	940
	941
	942	func (m ResultExtras) GetEventToken() EventToken {
	943	if m != nil {
	944	return m.EventToken
	945	}
	946	return nil
	947	}
	948
	949	func (m *ResultExtras) GetFresher() bool {
	950	if m != nil {
	951	return m.Fresher
	952	}
	953	return false
	954	}
	955
	956	// QueryResult is returned by Execute and ExecuteStream.
	957	//
	958	// As returned by Execute, len(fields) is always equal to len(row)
	959	// (for each row in rows).
	960	//
	961	// As returned by StreamExecute, the first QueryResult has the fields
	962	// set, and subsequent QueryResult have rows set. And as Execute,
	963	// len(QueryResult[0].fields) is always equal to len(row) (for each
	964	// row in rows for each QueryResult in QueryResult[1:]).
	965	type QueryResult struct {
	966	Fields []*Field `protobuf:"bytes,1,rep,name=fields" json:"fields,omitempty"`
	967	RowsAffected uint64 `protobuf:"varint,2,opt,name=rows_affected,json=rowsAffected" json:"rows_affected,omitempty"`
	968	InsertId uint64 `protobuf:"varint,3,opt,name=insert_id,json=insertId" json:"insert_id,omitempty"`
	969	Rows []*Row `protobuf:"bytes,4,rep,name=rows" json:"rows,omitempty"`
	970	Extras *ResultExtras `protobuf:"bytes,5,opt,name=extras" json:"extras,omitempty"`
	971	}
	972
	973	func (m QueryResult) Reset() { m = QueryResult{} }
	974	func (m *QueryResult) String() string { return "TODO" }
	975
	976
	977
	978	func (m QueryResult) GetFields() []Field {
	979	if m != nil {
	980	return m.Fields
	981	}
	982	return nil
	983	}
	984
	985	func (m *QueryResult) GetRowsAffected() uint64 {
	986	if m != nil {
	987	return m.RowsAffected
	988	}
	989	return 0
	990	}
	991
	992	func (m *QueryResult) GetInsertId() uint64 {
	993	if m != nil {
	994	return m.InsertId
	995	}
	996	return 0
	997	}
	998
	999	func (m QueryResult) GetRows() []Row {
	1000	if m != nil {
	1001	return m.Rows
	1002	}
	1003	return nil
	1004	}
	1005
	1006	func (m QueryResult) GetExtras() ResultExtras {
	1007	if m != nil {
	1008	return m.Extras
	1009	}
	1010	return nil
	1011	}
	1012
	1013	// StreamEvent describes a set of transformations that happened as a
	1014	// single transactional unit on a server. It is streamed back by the
	1015	// Update Stream calls.
	1016	type StreamEvent struct {
	1017	// The statements in this transaction.
	1018	Statements []*StreamEvent_Statement `protobuf:"bytes,1,rep,name=statements" json:"statements,omitempty"`
	1019	// The Event Token for this event.
	1020	EventToken *EventToken `protobuf:"bytes,2,opt,name=event_token,json=eventToken" json:"event_token,omitempty"`
	1021	}
	1022
	1023	func (m StreamEvent) Reset() { m = StreamEvent{} }
	1024	func (m *StreamEvent) String() string { return "TODO" }
	1025
	1026
	1027
	1028	func (m StreamEvent) GetStatements() []StreamEvent_Statement {
	1029	if m != nil {
	1030	return m.Statements
	1031	}
	1032	return nil
	1033	}
	1034
	1035	func (m StreamEvent) GetEventToken() EventToken {
	1036	if m != nil {
	1037	return m.EventToken
	1038	}
	1039	return nil
	1040	}
	1041
	1042	// One individual Statement in a transaction.
	1043	type StreamEvent_Statement struct {
	1044	Category StreamEvent_Statement_Category `protobuf:"varint,1,opt,name=category,enum=query.StreamEvent_Statement_Category" json:"category,omitempty"`
	1045	// table_name, primary_key_fields and primary_key_values are set for DML.
	1046	TableName string `protobuf:"bytes,2,opt,name=table_name,json=tableName" json:"table_name,omitempty"`
	1047	PrimaryKeyFields []*Field `protobuf:"bytes,3,rep,name=primary_key_fields,json=primaryKeyFields" json:"primary_key_fields,omitempty"`
	1048	PrimaryKeyValues []*Row `protobuf:"bytes,4,rep,name=primary_key_values,json=primaryKeyValues" json:"primary_key_values,omitempty"`
	1049	// sql is set for all queries.
	1050	// FIXME(alainjobart) we may not need it for DMLs.
	1051	Sql []byte `protobuf:"bytes,5,opt,name=sql,proto3" json:"sql,omitempty"`
	1052	}
	1053
	1054	func (m StreamEvent_Statement) Reset() { m = StreamEvent_Statement{} }
	1055	func (m *StreamEvent_Statement) String() string { return "TODO" }
	1056
	1057
	1058
	1059	func (m *StreamEvent_Statement) GetCategory() StreamEvent_Statement_Category {
	1060	if m != nil {
	1061	return m.Category
	1062	}
	1063	return StreamEvent_Statement_Error
	1064	}
	1065
	1066	func (m *StreamEvent_Statement) GetTableName() string {
	1067	if m != nil {
	1068	return m.TableName
	1069	}
	1070	return ""
	1071	}
	1072
	1073	func (m StreamEvent_Statement) GetPrimaryKeyFields() []Field {
	1074	if m != nil {
	1075	return m.PrimaryKeyFields
	1076	}
	1077	return nil
	1078	}
	1079
	1080	func (m StreamEvent_Statement) GetPrimaryKeyValues() []Row {
	1081	if m != nil {
	1082	return m.PrimaryKeyValues
	1083	}
	1084	return nil
	1085	}
	1086
	1087	func (m *StreamEvent_Statement) GetSql() []byte {
	1088	if m != nil {
	1089	return m.Sql
	1090	}
	1091	return nil
	1092	}
	1093
	1094
	1095	// ExecuteRequest is the payload to Execute
	1096	type ExecuteRequest struct {
	1097	EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
	1098	ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
	1099	Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
	1100	Query *BoundQuery `protobuf:"bytes,4,opt,name=query" json:"query,omitempty"`
	1101	TransactionId int64 `protobuf:"varint,5,opt,name=transaction_id,json=transactionId" json:"transaction_id,omitempty"`
	1102	Options *ExecuteOptions `protobuf:"bytes,6,opt,name=options" json:"options,omitempty"`
	1103	}
	1104
	1105	func (m ExecuteRequest) Reset() { m = ExecuteRequest{} }
	1106	func (m *ExecuteRequest) String() string { return "TODO" }
	1107
	1108
	1109
	1110	func (m ExecuteRequest) GetEffectiveCallerId() vtrpc.CallerID {
	1111	if m != nil {
	1112	return m.EffectiveCallerId
	1113	}
	1114	return nil
	1115	}
	1116
	1117	func (m ExecuteRequest) GetImmediateCallerId() VTGateCallerID {
	1118	if m != nil {
	1119	return m.ImmediateCallerId
	1120	}
	1121	return nil
	1122	}
	1123
	1124	func (m ExecuteRequest) GetTarget() Target {
	1125	if m != nil {
	1126	return m.Target
	1127	}
	1128	return nil
	1129	}
	1130
	1131	func (m ExecuteRequest) GetQuery() BoundQuery {
	1132	if m != nil {
	1133	return m.Query
	1134	}
	1135	return nil
	1136	}
	1137
	1138	func (m *ExecuteRequest) GetTransactionId() int64 {
	1139	if m != nil {
	1140	return m.TransactionId
	1141	}
	1142	return 0
	1143	}
	1144
	1145	func (m ExecuteRequest) GetOptions() ExecuteOptions {
	1146	if m != nil {
	1147	return m.Options
	1148	}
	1149	return nil
	1150	}
	1151
	1152	// ExecuteResponse is the returned value from Execute
	1153	type ExecuteResponse struct {
	1154	Result *QueryResult `protobuf:"bytes,1,opt,name=result" json:"result,omitempty"`
	1155	}
	1156
	1157	func (m ExecuteResponse) Reset() { m = ExecuteResponse{} }
	1158	func (m *ExecuteResponse) String() string { return "TODO" }
	1159
	1160
	1161
	1162	func (m ExecuteResponse) GetResult() QueryResult {
	1163	if m != nil {
	1164	return m.Result
	1165	}
	1166	return nil
	1167	}
	1168
	1169	// ResultWithError represents a query response
	1170	// in the form of result or error but not both.
	1171	// TODO: To be used in ExecuteBatchResponse and BeginExecuteBatchResponse.
	1172	type ResultWithError struct {
	1173	// error contains an query level error, only set if result is unset.
	1174	Error *vtrpc.RPCError `protobuf:"bytes,1,opt,name=error" json:"error,omitempty"`
	1175	// result contains the query result, only set if error is unset.
	1176	Result *QueryResult `protobuf:"bytes,2,opt,name=result" json:"result,omitempty"`
	1177	}
	1178
	1179	func (m ResultWithError) Reset() { m = ResultWithError{} }
	1180	func (m *ResultWithError) String() string { return "TODO" }
	1181
	1182
	1183
	1184	func (m ResultWithError) GetError() vtrpc.RPCError {
	1185	if m != nil {
	1186	return m.Error
	1187	}
	1188	return nil
	1189	}
	1190
	1191	func (m ResultWithError) GetResult() QueryResult {
	1192	if m != nil {
	1193	return m.Result
	1194	}
	1195	return nil
	1196	}
	1197
	1198	// ExecuteBatchRequest is the payload to ExecuteBatch
	1199	type ExecuteBatchRequest struct {
	1200	EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
	1201	ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
	1202	Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
	1203	Queries []*BoundQuery `protobuf:"bytes,4,rep,name=queries" json:"queries,omitempty"`
	1204	AsTransaction bool `protobuf:"varint,5,opt,name=as_transaction,json=asTransaction" json:"as_transaction,omitempty"`
	1205	TransactionId int64 `protobuf:"varint,6,opt,name=transaction_id,json=transactionId" json:"transaction_id,omitempty"`
	1206	Options *ExecuteOptions `protobuf:"bytes,7,opt,name=options" json:"options,omitempty"`
	1207	}
	1208
	1209	func (m ExecuteBatchRequest) Reset() { m = ExecuteBatchRequest{} }
	1210	func (m *ExecuteBatchRequest) String() string { return "TODO" }
	1211
	1212
	1213
	1214	func (m ExecuteBatchRequest) GetEffectiveCallerId() vtrpc.CallerID {
	1215	if m != nil {
	1216	return m.EffectiveCallerId
	1217	}
	1218	return nil
	1219	}
	1220
	1221	func (m ExecuteBatchRequest) GetImmediateCallerId() VTGateCallerID {
	1222	if m != nil {
	1223	return m.ImmediateCallerId
	1224	}
	1225	return nil
	1226	}
	1227
	1228	func (m ExecuteBatchRequest) GetTarget() Target {
	1229	if m != nil {
	1230	return m.Target
	1231	}
	1232	return nil
	1233	}
	1234
	1235	func (m ExecuteBatchRequest) GetQueries() []BoundQuery {
	1236	if m != nil {
	1237	return m.Queries
	1238	}
	1239	return nil
	1240	}
	1241
	1242	func (m *ExecuteBatchRequest) GetAsTransaction() bool {
	1243	if m != nil {
	1244	return m.AsTransaction
	1245	}
	1246	return false
	1247	}
	1248
	1249	func (m *ExecuteBatchRequest) GetTransactionId() int64 {
	1250	if m != nil {
	1251	return m.TransactionId
	1252	}
	1253	return 0
	1254	}
	1255
	1256	func (m ExecuteBatchRequest) GetOptions() ExecuteOptions {
	1257	if m != nil {
	1258	return m.Options
	1259	}
	1260	return nil
	1261	}
	1262
	1263	// ExecuteBatchResponse is the returned value from ExecuteBatch
	1264	type ExecuteBatchResponse struct {
	1265	Results []*QueryResult `protobuf:"bytes,1,rep,name=results" json:"results,omitempty"`
	1266	}
	1267
	1268	func (m ExecuteBatchResponse) Reset() { m = ExecuteBatchResponse{} }
	1269	func (m *ExecuteBatchResponse) String() string { return "TODO" }
	1270
	1271
	1272
	1273	func (m ExecuteBatchResponse) GetResults() []QueryResult {
	1274	if m != nil {
	1275	return m.Results
	1276	}
	1277	return nil
	1278	}
	1279
	1280	// StreamExecuteRequest is the payload to StreamExecute
	1281	type StreamExecuteRequest struct {
	1282	EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
	1283	ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
	1284	Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
	1285	Query *BoundQuery `protobuf:"bytes,4,opt,name=query" json:"query,omitempty"`
	1286	Options *ExecuteOptions `protobuf:"bytes,5,opt,name=options" json:"options,omitempty"`
	1287	}
	1288
	1289	func (m StreamExecuteRequest) Reset() { m = StreamExecuteRequest{} }
	1290	func (m *StreamExecuteRequest) String() string { return "TODO" }
	1291
	1292
	1293
	1294	func (m StreamExecuteRequest) GetEffectiveCallerId() vtrpc.CallerID {
	1295	if m != nil {
	1296	return m.EffectiveCallerId
	1297	}
	1298	return nil
	1299	}
	1300
	1301	func (m StreamExecuteRequest) GetImmediateCallerId() VTGateCallerID {
	1302	if m != nil {
	1303	return m.ImmediateCallerId
	1304	}
	1305	return nil
	1306	}
	1307
	1308	func (m StreamExecuteRequest) GetTarget() Target {
	1309	if m != nil {
	1310	return m.Target
	1311	}
	1312	return nil
	1313	}
	1314
	1315	func (m StreamExecuteRequest) GetQuery() BoundQuery {
	1316	if m != nil {
	1317	return m.Query
	1318	}
	1319	return nil
	1320	}
	1321
	1322	func (m StreamExecuteRequest) GetOptions() ExecuteOptions {
	1323	if m != nil {
	1324	return m.Options
	1325	}
	1326	return nil
	1327	}
	1328
	1329	// StreamExecuteResponse is the returned value from StreamExecute
	1330	type StreamExecuteResponse struct {
	1331	Result *QueryResult `protobuf:"bytes,1,opt,name=result" json:"result,omitempty"`
	1332	}
	1333
	1334	func (m StreamExecuteResponse) Reset() { m = StreamExecuteResponse{} }
	1335	func (m *StreamExecuteResponse) String() string { return "TODO" }
	1336
	1337
	1338
	1339	func (m StreamExecuteResponse) GetResult() QueryResult {
	1340	if m != nil {
	1341	return m.Result
	1342	}
	1343	return nil
	1344	}
	1345
	1346	// BeginRequest is the payload to Begin
	1347	type BeginRequest struct {
	1348	EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
	1349	ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
	1350	Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
	1351	Options *ExecuteOptions `protobuf:"bytes,4,opt,name=options" json:"options,omitempty"`
	1352	}
	1353
	1354	func (m BeginRequest) Reset() { m = BeginRequest{} }
	1355	func (m *BeginRequest) String() string { return "TODO" }
	1356
	1357
	1358
	1359	func (m BeginRequest) GetEffectiveCallerId() vtrpc.CallerID {
	1360	if m != nil {
	1361	return m.EffectiveCallerId
	1362	}
	1363	return nil
	1364	}
	1365
	1366	func (m BeginRequest) GetImmediateCallerId() VTGateCallerID {
	1367	if m != nil {
	1368	return m.ImmediateCallerId
	1369	}
	1370	return nil
	1371	}
	1372
	1373	func (m BeginRequest) GetTarget() Target {
	1374	if m != nil {
	1375	return m.Target
	1376	}
	1377	return nil
	1378	}
	1379
	1380	func (m BeginRequest) GetOptions() ExecuteOptions {
	1381	if m != nil {
	1382	return m.Options
	1383	}
	1384	return nil
	1385	}
	1386
	1387	// BeginResponse is the returned value from Begin
	1388	type BeginResponse struct {
	1389	TransactionId int64 `protobuf:"varint,1,opt,name=transaction_id,json=transactionId" json:"transaction_id,omitempty"`
	1390	}
	1391
	1392	func (m BeginResponse) Reset() { m = BeginResponse{} }
	1393	func (m *BeginResponse) String() string { return "TODO" }
	1394
	1395
	1396
	1397	func (m *BeginResponse) GetTransactionId() int64 {
	1398	if m != nil {
	1399	return m.TransactionId
	1400	}
	1401	return 0
	1402	}
	1403
	1404	// CommitRequest is the payload to Commit
	1405	type CommitRequest struct {
	1406	EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
	1407	ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
	1408	Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
	1409	TransactionId int64 `protobuf:"varint,4,opt,name=transaction_id,json=transactionId" json:"transaction_id,omitempty"`
	1410	}
	1411
	1412	func (m CommitRequest) Reset() { m = CommitRequest{} }
	1413	func (m *CommitRequest) String() string { return "TODO" }
	1414
	1415
	1416
	1417	func (m CommitRequest) GetEffectiveCallerId() vtrpc.CallerID {
	1418	if m != nil {
	1419	return m.EffectiveCallerId
	1420	}
	1421	return nil
	1422	}
	1423
	1424	func (m CommitRequest) GetImmediateCallerId() VTGateCallerID {
	1425	if m != nil {
	1426	return m.ImmediateCallerId
	1427	}
	1428	return nil
	1429	}
	1430
	1431	func (m CommitRequest) GetTarget() Target {
	1432	if m != nil {
	1433	return m.Target
	1434	}
	1435	return nil
	1436	}
	1437
	1438	func (m *CommitRequest) GetTransactionId() int64 {
	1439	if m != nil {
	1440	return m.TransactionId
	1441	}
	1442	return 0
	1443	}
	1444
	1445	// CommitResponse is the returned value from Commit
	1446	type CommitResponse struct {
	1447	}
	1448
	1449	func (m CommitResponse) Reset() { m = CommitResponse{} }
	1450	func (m *CommitResponse) String() string { return "TODO" }
	1451
	1452
	1453
	1454	// RollbackRequest is the payload to Rollback
	1455	type RollbackRequest struct {
	1456	EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
	1457	ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
	1458	Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
	1459	TransactionId int64 `protobuf:"varint,4,opt,name=transaction_id,json=transactionId" json:"transaction_id,omitempty"`
	1460	}
	1461
	1462	func (m RollbackRequest) Reset() { m = RollbackRequest{} }
	1463	func (m *RollbackRequest) String() string { return "TODO" }
	1464
	1465
	1466
	1467	func (m RollbackRequest) GetEffectiveCallerId() vtrpc.CallerID {
	1468	if m != nil {
	1469	return m.EffectiveCallerId
	1470	}
	1471	return nil
	1472	}
	1473
	1474	func (m RollbackRequest) GetImmediateCallerId() VTGateCallerID {
	1475	if m != nil {
	1476	return m.ImmediateCallerId
	1477	}
	1478	return nil
	1479	}
	1480
	1481	func (m RollbackRequest) GetTarget() Target {
	1482	if m != nil {
	1483	return m.Target
	1484	}
	1485	return nil
	1486	}
	1487
	1488	func (m *RollbackRequest) GetTransactionId() int64 {
	1489	if m != nil {
	1490	return m.TransactionId
	1491	}
	1492	return 0
	1493	}
	1494
	1495	// RollbackResponse is the returned value from Rollback
	1496	type RollbackResponse struct {
	1497	}
	1498
	1499	func (m RollbackResponse) Reset() { m = RollbackResponse{} }
	1500	func (m *RollbackResponse) String() string { return "TODO" }
	1501
	1502
	1503
	1504	// PrepareRequest is the payload to Prepare
	1505	type PrepareRequest struct {
	1506	EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
	1507	ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
	1508	Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
	1509	TransactionId int64 `protobuf:"varint,4,opt,name=transaction_id,json=transactionId" json:"transaction_id,omitempty"`
	1510	Dtid string `protobuf:"bytes,5,opt,name=dtid" json:"dtid,omitempty"`
	1511	}
	1512
	1513	func (m PrepareRequest) Reset() { m = PrepareRequest{} }
	1514	func (m *PrepareRequest) String() string { return "TODO" }
	1515
	1516
	1517
	1518	func (m PrepareRequest) GetEffectiveCallerId() vtrpc.CallerID {
	1519	if m != nil {
	1520	return m.EffectiveCallerId
	1521	}
	1522	return nil
	1523	}
	1524
	1525	func (m PrepareRequest) GetImmediateCallerId() VTGateCallerID {
	1526	if m != nil {
	1527	return m.ImmediateCallerId
	1528	}
	1529	return nil
	1530	}
	1531
	1532	func (m PrepareRequest) GetTarget() Target {
	1533	if m != nil {
	1534	return m.Target
	1535	}
	1536	return nil
	1537	}
	1538
	1539	func (m *PrepareRequest) GetTransactionId() int64 {
	1540	if m != nil {
	1541	return m.TransactionId
	1542	}
	1543	return 0
	1544	}
	1545
	1546	func (m *PrepareRequest) GetDtid() string {
	1547	if m != nil {
	1548	return m.Dtid
	1549	}
	1550	return ""
	1551	}
	1552
	1553	// PrepareResponse is the returned value from Prepare
	1554	type PrepareResponse struct {
	1555	}
	1556
	1557	func (m PrepareResponse) Reset() { m = PrepareResponse{} }
	1558	func (m *PrepareResponse) String() string { return "TODO" }
	1559
	1560
	1561
	1562	// CommitPreparedRequest is the payload to CommitPrepared
	1563	type CommitPreparedRequest struct {
	1564	EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
	1565	ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
	1566	Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
	1567	Dtid string `protobuf:"bytes,4,opt,name=dtid" json:"dtid,omitempty"`
	1568	}
	1569
	1570	func (m CommitPreparedRequest) Reset() { m = CommitPreparedRequest{} }
	1571	func (m *CommitPreparedRequest) String() string { return "TODO" }
	1572
	1573
	1574
	1575	func (m CommitPreparedRequest) GetEffectiveCallerId() vtrpc.CallerID {
	1576	if m != nil {
	1577	return m.EffectiveCallerId
	1578	}
	1579	return nil
	1580	}
	1581
	1582	func (m CommitPreparedRequest) GetImmediateCallerId() VTGateCallerID {
	1583	if m != nil {
	1584	return m.ImmediateCallerId
	1585	}
	1586	return nil
	1587	}
	1588
	1589	func (m CommitPreparedRequest) GetTarget() Target {
	1590	if m != nil {
	1591	return m.Target
	1592	}
	1593	return nil
	1594	}
	1595
	1596	func (m *CommitPreparedRequest) GetDtid() string {
	1597	if m != nil {
	1598	return m.Dtid
	1599	}
	1600	return ""
	1601	}
	1602
	1603	// CommitPreparedResponse is the returned value from CommitPrepared
	1604	type CommitPreparedResponse struct {
	1605	}
	1606
	1607	func (m CommitPreparedResponse) Reset() { m = CommitPreparedResponse{} }
	1608	func (m *CommitPreparedResponse) String() string { return "TODO" }
	1609
	1610
	1611
	1612	// RollbackPreparedRequest is the payload to RollbackPrepared
	1613	type RollbackPreparedRequest struct {
	1614	EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
	1615	ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
	1616	Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
	1617	TransactionId int64 `protobuf:"varint,4,opt,name=transaction_id,json=transactionId" json:"transaction_id,omitempty"`
	1618	Dtid string `protobuf:"bytes,5,opt,name=dtid" json:"dtid,omitempty"`
	1619	}
	1620
	1621	func (m RollbackPreparedRequest) Reset() { m = RollbackPreparedRequest{} }
	1622	func (m *RollbackPreparedRequest) String() string { return "TODO" }
	1623
	1624
	1625
	1626	func (m RollbackPreparedRequest) GetEffectiveCallerId() vtrpc.CallerID {
	1627	if m != nil {
	1628	return m.EffectiveCallerId
	1629	}
	1630	return nil
	1631	}
	1632
	1633	func (m RollbackPreparedRequest) GetImmediateCallerId() VTGateCallerID {
	1634	if m != nil {
	1635	return m.ImmediateCallerId
	1636	}
	1637	return nil
	1638	}
	1639
	1640	func (m RollbackPreparedRequest) GetTarget() Target {
	1641	if m != nil {
	1642	return m.Target
	1643	}
	1644	return nil
	1645	}
	1646
	1647	func (m *RollbackPreparedRequest) GetTransactionId() int64 {
	1648	if m != nil {
	1649	return m.TransactionId
	1650	}
	1651	return 0
	1652	}
	1653
	1654	func (m *RollbackPreparedRequest) GetDtid() string {
	1655	if m != nil {
	1656	return m.Dtid
	1657	}
	1658	return ""
	1659	}
	1660
	1661	// RollbackPreparedResponse is the returned value from RollbackPrepared
	1662	type RollbackPreparedResponse struct {
	1663	}
	1664
	1665	func (m RollbackPreparedResponse) Reset() { m = RollbackPreparedResponse{} }
	1666	func (m *RollbackPreparedResponse) String() string { return "TODO" }
	1667
	1668
	1669
	1670	// CreateTransactionRequest is the payload to CreateTransaction
	1671	type CreateTransactionRequest struct {
	1672	EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
	1673	ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
	1674	Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
	1675	Dtid string `protobuf:"bytes,4,opt,name=dtid" json:"dtid,omitempty"`
	1676	Participants []*Target `protobuf:"bytes,5,rep,name=participants" json:"participants,omitempty"`
	1677	}
	1678
	1679	func (m CreateTransactionRequest) Reset() { m = CreateTransactionRequest{} }
	1680	func (m *CreateTransactionRequest) String() string { return "TODO" }
	1681
	1682
	1683
	1684	func (m CreateTransactionRequest) GetEffectiveCallerId() vtrpc.CallerID {
	1685	if m != nil {
	1686	return m.EffectiveCallerId
	1687	}
	1688	return nil
	1689	}
	1690
	1691	func (m CreateTransactionRequest) GetImmediateCallerId() VTGateCallerID {
	1692	if m != nil {
	1693	return m.ImmediateCallerId
	1694	}
	1695	return nil
	1696	}
	1697
	1698	func (m CreateTransactionRequest) GetTarget() Target {
	1699	if m != nil {
	1700	return m.Target
	1701	}
	1702	return nil
	1703	}
	1704
	1705	func (m *CreateTransactionRequest) GetDtid() string {
	1706	if m != nil {
	1707	return m.Dtid
	1708	}
	1709	return ""
	1710	}
	1711
	1712	func (m CreateTransactionRequest) GetParticipants() []Target {
	1713	if m != nil {
	1714	return m.Participants
	1715	}
	1716	return nil
	1717	}
	1718
	1719	// CreateTransactionResponse is the returned value from CreateTransaction
	1720	type CreateTransactionResponse struct {
	1721	}
	1722
	1723	func (m CreateTransactionResponse) Reset() { m = CreateTransactionResponse{} }
	1724	func (m *CreateTransactionResponse) String() string { return "TODO" }
	1725
	1726
	1727
	1728	// StartCommitRequest is the payload to StartCommit
	1729	type StartCommitRequest struct {
	1730	EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
	1731	ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
	1732	Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
	1733	TransactionId int64 `protobuf:"varint,4,opt,name=transaction_id,json=transactionId" json:"transaction_id,omitempty"`
	1734	Dtid string `protobuf:"bytes,5,opt,name=dtid" json:"dtid,omitempty"`
	1735	}
	1736
	1737	func (m StartCommitRequest) Reset() { m = StartCommitRequest{} }
	1738	func (m *StartCommitRequest) String() string { return "TODO" }
	1739
	1740
	1741
	1742	func (m StartCommitRequest) GetEffectiveCallerId() vtrpc.CallerID {
	1743	if m != nil {
	1744	return m.EffectiveCallerId
	1745	}
	1746	return nil
	1747	}
	1748
	1749	func (m StartCommitRequest) GetImmediateCallerId() VTGateCallerID {
	1750	if m != nil {
	1751	return m.ImmediateCallerId
	1752	}
	1753	return nil
	1754	}
	1755
	1756	func (m StartCommitRequest) GetTarget() Target {
	1757	if m != nil {
	1758	return m.Target
	1759	}
	1760	return nil
	1761	}
	1762
	1763	func (m *StartCommitRequest) GetTransactionId() int64 {
	1764	if m != nil {
	1765	return m.TransactionId
	1766	}
	1767	return 0
	1768	}
	1769
	1770	func (m *StartCommitRequest) GetDtid() string {
	1771	if m != nil {
	1772	return m.Dtid
	1773	}
	1774	return ""
	1775	}
	1776
	1777	// StartCommitResponse is the returned value from StartCommit
	1778	type StartCommitResponse struct {
	1779	}
	1780
	1781	func (m StartCommitResponse) Reset() { m = StartCommitResponse{} }
	1782	func (m *StartCommitResponse) String() string { return "TODO" }
	1783
	1784
	1785
	1786	// SetRollbackRequest is the payload to SetRollback
	1787	type SetRollbackRequest struct {
	1788	EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
	1789	ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
	1790	Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
	1791	TransactionId int64 `protobuf:"varint,4,opt,name=transaction_id,json=transactionId" json:"transaction_id,omitempty"`
	1792	Dtid string `protobuf:"bytes,5,opt,name=dtid" json:"dtid,omitempty"`
	1793	}
	1794
	1795	func (m SetRollbackRequest) Reset() { m = SetRollbackRequest{} }
	1796	func (m *SetRollbackRequest) String() string { return "TODO" }
	1797
	1798
	1799
	1800	func (m SetRollbackRequest) GetEffectiveCallerId() vtrpc.CallerID {
	1801	if m != nil {
	1802	return m.EffectiveCallerId
	1803	}
	1804	return nil
	1805	}
	1806
	1807	func (m SetRollbackRequest) GetImmediateCallerId() VTGateCallerID {
	1808	if m != nil {
	1809	return m.ImmediateCallerId
	1810	}
	1811	return nil
	1812	}
	1813
	1814	func (m SetRollbackRequest) GetTarget() Target {
	1815	if m != nil {
	1816	return m.Target
	1817	}
	1818	return nil
	1819	}
	1820
	1821	func (m *SetRollbackRequest) GetTransactionId() int64 {
	1822	if m != nil {
	1823	return m.TransactionId
	1824	}
	1825	return 0
	1826	}
	1827
	1828	func (m *SetRollbackRequest) GetDtid() string {
	1829	if m != nil {
	1830	return m.Dtid
	1831	}
	1832	return ""
	1833	}
	1834
	1835	// SetRollbackResponse is the returned value from SetRollback
	1836	type SetRollbackResponse struct {
	1837	}
	1838
	1839	func (m SetRollbackResponse) Reset() { m = SetRollbackResponse{} }
	1840	func (m *SetRollbackResponse) String() string { return "TODO" }
	1841
	1842
	1843
	1844	// ConcludeTransactionRequest is the payload to ConcludeTransaction
	1845	type ConcludeTransactionRequest struct {
	1846	EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
	1847	ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
	1848	Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
	1849	Dtid string `protobuf:"bytes,4,opt,name=dtid" json:"dtid,omitempty"`
	1850	}
	1851
	1852	func (m ConcludeTransactionRequest) Reset() { m = ConcludeTransactionRequest{} }
	1853	func (m *ConcludeTransactionRequest) String() string { return "TODO" }
	1854
	1855
	1856
	1857	func (m ConcludeTransactionRequest) GetEffectiveCallerId() vtrpc.CallerID {
	1858	if m != nil {
	1859	return m.EffectiveCallerId
	1860	}
	1861	return nil
	1862	}
	1863
	1864	func (m ConcludeTransactionRequest) GetImmediateCallerId() VTGateCallerID {
	1865	if m != nil {
	1866	return m.ImmediateCallerId
	1867	}
	1868	return nil
	1869	}
	1870
	1871	func (m ConcludeTransactionRequest) GetTarget() Target {
	1872	if m != nil {
	1873	return m.Target
	1874	}
	1875	return nil
	1876	}
	1877
	1878	func (m *ConcludeTransactionRequest) GetDtid() string {
	1879	if m != nil {
	1880	return m.Dtid
	1881	}
	1882	return ""
	1883	}
	1884
	1885	// ConcludeTransactionResponse is the returned value from ConcludeTransaction
	1886	type ConcludeTransactionResponse struct {
	1887	}
	1888
	1889	func (m ConcludeTransactionResponse) Reset() { m = ConcludeTransactionResponse{} }
	1890	func (m *ConcludeTransactionResponse) String() string { return "TODO" }
	1891
	1892
	1893
	1894	// ReadTransactionRequest is the payload to ReadTransaction
	1895	type ReadTransactionRequest struct {
	1896	EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
	1897	ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
	1898	Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
	1899	Dtid string `protobuf:"bytes,4,opt,name=dtid" json:"dtid,omitempty"`
	1900	}
	1901
	1902	func (m ReadTransactionRequest) Reset() { m = ReadTransactionRequest{} }
	1903	func (m *ReadTransactionRequest) String() string { return "TODO" }
	1904
	1905
	1906
	1907	func (m ReadTransactionRequest) GetEffectiveCallerId() vtrpc.CallerID {
	1908	if m != nil {
	1909	return m.EffectiveCallerId
	1910	}
	1911	return nil
	1912	}
	1913
	1914	func (m ReadTransactionRequest) GetImmediateCallerId() VTGateCallerID {
	1915	if m != nil {
	1916	return m.ImmediateCallerId
	1917	}
	1918	return nil
	1919	}
	1920
	1921	func (m ReadTransactionRequest) GetTarget() Target {
	1922	if m != nil {
	1923	return m.Target
	1924	}
	1925	return nil
	1926	}
	1927
	1928	func (m *ReadTransactionRequest) GetDtid() string {
	1929	if m != nil {
	1930	return m.Dtid
	1931	}
	1932	return ""
	1933	}
	1934
	1935	// ReadTransactionResponse is the returned value from ReadTransaction
	1936	type ReadTransactionResponse struct {
	1937	Metadata *TransactionMetadata `protobuf:"bytes,1,opt,name=metadata" json:"metadata,omitempty"`
	1938	}
	1939
	1940	func (m ReadTransactionResponse) Reset() { m = ReadTransactionResponse{} }
	1941	func (m *ReadTransactionResponse) String() string { return "TODO" }
	1942
	1943
	1944
	1945	func (m ReadTransactionResponse) GetMetadata() TransactionMetadata {
	1946	if m != nil {
	1947	return m.Metadata
	1948	}
	1949	return nil
	1950	}
	1951
	1952	// BeginExecuteRequest is the payload to BeginExecute
	1953	type BeginExecuteRequest struct {
	1954	EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
	1955	ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
	1956	Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
	1957	Query *BoundQuery `protobuf:"bytes,4,opt,name=query" json:"query,omitempty"`
	1958	Options *ExecuteOptions `protobuf:"bytes,5,opt,name=options" json:"options,omitempty"`
	1959	}
	1960
	1961	func (m BeginExecuteRequest) Reset() { m = BeginExecuteRequest{} }
	1962	func (m *BeginExecuteRequest) String() string { return "TODO" }
	1963
	1964
	1965
	1966	func (m BeginExecuteRequest) GetEffectiveCallerId() vtrpc.CallerID {
	1967	if m != nil {
	1968	return m.EffectiveCallerId
	1969	}
	1970	return nil
	1971	}
	1972
	1973	func (m BeginExecuteRequest) GetImmediateCallerId() VTGateCallerID {
	1974	if m != nil {
	1975	return m.ImmediateCallerId
	1976	}
	1977	return nil
	1978	}
	1979
	1980	func (m BeginExecuteRequest) GetTarget() Target {
	1981	if m != nil {
	1982	return m.Target
	1983	}
	1984	return nil
	1985	}
	1986
	1987	func (m BeginExecuteRequest) GetQuery() BoundQuery {
	1988	if m != nil {
	1989	return m.Query
	1990	}
	1991	return nil
	1992	}
	1993
	1994	func (m BeginExecuteRequest) GetOptions() ExecuteOptions {
	1995	if m != nil {
	1996	return m.Options
	1997	}
	1998	return nil
	1999	}
	2000
	2001	// BeginExecuteResponse is the returned value from BeginExecute
	2002	type BeginExecuteResponse struct {
	2003	// error contains an application level error if necessary. Note the
	2004	// transaction_id may be set, even when an error is returned, if the begin
	2005	// worked but the execute failed.
	2006	Error *vtrpc.RPCError `protobuf:"bytes,1,opt,name=error" json:"error,omitempty"`
	2007	Result *QueryResult `protobuf:"bytes,2,opt,name=result" json:"result,omitempty"`
	2008	// transaction_id might be non-zero even if an error is present.
	2009	TransactionId int64 `protobuf:"varint,3,opt,name=transaction_id,json=transactionId" json:"transaction_id,omitempty"`
	2010	}
	2011
	2012	func (m BeginExecuteResponse) Reset() { m = BeginExecuteResponse{} }
	2013	func (m *BeginExecuteResponse) String() string { return "TODO" }
	2014
	2015
	2016
	2017	func (m BeginExecuteResponse) GetError() vtrpc.RPCError {
	2018	if m != nil {
	2019	return m.Error
	2020	}
	2021	return nil
	2022	}
	2023
	2024	func (m BeginExecuteResponse) GetResult() QueryResult {
	2025	if m != nil {
	2026	return m.Result
	2027	}
	2028	return nil
	2029	}
	2030
	2031	func (m *BeginExecuteResponse) GetTransactionId() int64 {
	2032	if m != nil {
	2033	return m.TransactionId
	2034	}
	2035	return 0
	2036	}
	2037
	2038	// BeginExecuteBatchRequest is the payload to BeginExecuteBatch
	2039	type BeginExecuteBatchRequest struct {
	2040	EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
	2041	ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
	2042	Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
	2043	Queries []*BoundQuery `protobuf:"bytes,4,rep,name=queries" json:"queries,omitempty"`
	2044	AsTransaction bool `protobuf:"varint,5,opt,name=as_transaction,json=asTransaction" json:"as_transaction,omitempty"`
	2045	Options *ExecuteOptions `protobuf:"bytes,6,opt,name=options" json:"options,omitempty"`
	2046	}
	2047
	2048	func (m BeginExecuteBatchRequest) Reset() { m = BeginExecuteBatchRequest{} }
	2049	func (m *BeginExecuteBatchRequest) String() string { return "TODO" }
	2050
	2051
	2052
	2053	func (m BeginExecuteBatchRequest) GetEffectiveCallerId() vtrpc.CallerID {
	2054	if m != nil {
	2055	return m.EffectiveCallerId
	2056	}
	2057	return nil
	2058	}
	2059
	2060	func (m BeginExecuteBatchRequest) GetImmediateCallerId() VTGateCallerID {
	2061	if m != nil {
	2062	return m.ImmediateCallerId
	2063	}
	2064	return nil
	2065	}
	2066
	2067	func (m BeginExecuteBatchRequest) GetTarget() Target {
	2068	if m != nil {
	2069	return m.Target
	2070	}
	2071	return nil
	2072	}
	2073
	2074	func (m BeginExecuteBatchRequest) GetQueries() []BoundQuery {
	2075	if m != nil {
	2076	return m.Queries
	2077	}
	2078	return nil
	2079	}
	2080
	2081	func (m *BeginExecuteBatchRequest) GetAsTransaction() bool {
	2082	if m != nil {
	2083	return m.AsTransaction
	2084	}
	2085	return false
	2086	}
	2087
	2088	func (m BeginExecuteBatchRequest) GetOptions() ExecuteOptions {
	2089	if m != nil {
	2090	return m.Options
	2091	}
	2092	return nil
	2093	}
	2094
	2095	// BeginExecuteBatchResponse is the returned value from BeginExecuteBatch
	2096	type BeginExecuteBatchResponse struct {
	2097	// error contains an application level error if necessary. Note the
	2098	// transaction_id may be set, even when an error is returned, if the begin
	2099	// worked but the execute failed.
	2100	Error *vtrpc.RPCError `protobuf:"bytes,1,opt,name=error" json:"error,omitempty"`
	2101	Results []*QueryResult `protobuf:"bytes,2,rep,name=results" json:"results,omitempty"`
	2102	// transaction_id might be non-zero even if an error is present.
	2103	TransactionId int64 `protobuf:"varint,3,opt,name=transaction_id,json=transactionId" json:"transaction_id,omitempty"`
	2104	}
	2105
	2106	func (m BeginExecuteBatchResponse) Reset() { m = BeginExecuteBatchResponse{} }
	2107	func (m *BeginExecuteBatchResponse) String() string { return "TODO" }
	2108
	2109
	2110
	2111	func (m BeginExecuteBatchResponse) GetError() vtrpc.RPCError {
	2112	if m != nil {
	2113	return m.Error
	2114	}
	2115	return nil
	2116	}
	2117
	2118	func (m BeginExecuteBatchResponse) GetResults() []QueryResult {
	2119	if m != nil {
	2120	return m.Results
	2121	}
	2122	return nil
	2123	}
	2124
	2125	func (m *BeginExecuteBatchResponse) GetTransactionId() int64 {
	2126	if m != nil {
	2127	return m.TransactionId
	2128	}
	2129	return 0
	2130	}
	2131
	2132	// MessageStreamRequest is the request payload for MessageStream.
	2133	type MessageStreamRequest struct {
	2134	EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
	2135	ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
	2136	Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
	2137	// name is the message table name.
	2138	Name string `protobuf:"bytes,4,opt,name=name" json:"name,omitempty"`
	2139	}
	2140
	2141	func (m MessageStreamRequest) Reset() { m = MessageStreamRequest{} }
	2142	func (m *MessageStreamRequest) String() string { return "TODO" }
	2143
	2144
	2145
	2146	func (m MessageStreamRequest) GetEffectiveCallerId() vtrpc.CallerID {
	2147	if m != nil {
	2148	return m.EffectiveCallerId
	2149	}
	2150	return nil
	2151	}
	2152
	2153	func (m MessageStreamRequest) GetImmediateCallerId() VTGateCallerID {
	2154	if m != nil {
	2155	return m.ImmediateCallerId
	2156	}
	2157	return nil
	2158	}
	2159
	2160	func (m MessageStreamRequest) GetTarget() Target {
	2161	if m != nil {
	2162	return m.Target
	2163	}
	2164	return nil
	2165	}
	2166
	2167	func (m *MessageStreamRequest) GetName() string {
	2168	if m != nil {
	2169	return m.Name
	2170	}
	2171	return ""
	2172	}
	2173
	2174	// MessageStreamResponse is a response for MessageStream.
	2175	type MessageStreamResponse struct {
	2176	Result *QueryResult `protobuf:"bytes,1,opt,name=result" json:"result,omitempty"`
	2177	}
	2178
	2179	func (m MessageStreamResponse) Reset() { m = MessageStreamResponse{} }
	2180	func (m *MessageStreamResponse) String() string { return "TODO" }
	2181
	2182
	2183
	2184	func (m MessageStreamResponse) GetResult() QueryResult {
	2185	if m != nil {
	2186	return m.Result
	2187	}
	2188	return nil
	2189	}
	2190
	2191	// MessageAckRequest is the request payload for MessageAck.
	2192	type MessageAckRequest struct {
	2193	EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
	2194	ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
	2195	Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
	2196	// name is the message table name.
	2197	Name string `protobuf:"bytes,4,opt,name=name" json:"name,omitempty"`
	2198	Ids []*Value `protobuf:"bytes,5,rep,name=ids" json:"ids,omitempty"`
	2199	}
	2200
	2201	func (m MessageAckRequest) Reset() { m = MessageAckRequest{} }
	2202	func (m *MessageAckRequest) String() string { return "TODO" }
	2203
	2204
	2205
	2206	func (m MessageAckRequest) GetEffectiveCallerId() vtrpc.CallerID {
	2207	if m != nil {
	2208	return m.EffectiveCallerId
	2209	}
	2210	return nil
	2211	}
	2212
	2213	func (m MessageAckRequest) GetImmediateCallerId() VTGateCallerID {
	2214	if m != nil {
	2215	return m.ImmediateCallerId
	2216	}
	2217	return nil
	2218	}
	2219
	2220	func (m MessageAckRequest) GetTarget() Target {
	2221	if m != nil {
	2222	return m.Target
	2223	}
	2224	return nil
	2225	}
	2226
	2227	func (m *MessageAckRequest) GetName() string {
	2228	if m != nil {
	2229	return m.Name
	2230	}
	2231	return ""
	2232	}
	2233
	2234	func (m MessageAckRequest) GetIds() []Value {
	2235	if m != nil {
	2236	return m.Ids
	2237	}
	2238	return nil
	2239	}
	2240
	2241	// MessageAckResponse is the response for MessageAck.
	2242	type MessageAckResponse struct {
	2243	// result contains the result of the ack operation.
	2244	// Since this acts like a DML, only
	2245	// RowsAffected is returned in the result.
	2246	Result *QueryResult `protobuf:"bytes,1,opt,name=result" json:"result,omitempty"`
	2247	}
	2248
	2249	func (m MessageAckResponse) Reset() { m = MessageAckResponse{} }
	2250	func (m *MessageAckResponse) String() string { return "TODO" }
	2251
	2252
	2253
	2254	func (m MessageAckResponse) GetResult() QueryResult {
	2255	if m != nil {
	2256	return m.Result
	2257	}
	2258	return nil
	2259	}
	2260
	2261	// SplitQueryRequest is the payload for SplitQuery sent by VTGate to a VTTablet.
	2262	// See vtgate.SplitQueryRequest for more details.
	2263	type SplitQueryRequest struct {
	2264	EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
	2265	ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
	2266	Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
	2267	Query *BoundQuery `protobuf:"bytes,4,opt,name=query" json:"query,omitempty"`
	2268	SplitColumn []string `protobuf:"bytes,5,rep,name=split_column,json=splitColumn" json:"split_column,omitempty"`
	2269	// Exactly one of the following must be nonzero.
	2270	SplitCount int64 `protobuf:"varint,6,opt,name=split_count,json=splitCount" json:"split_count,omitempty"`
	2271	NumRowsPerQueryPart int64 `protobuf:"varint,8,opt,name=num_rows_per_query_part,json=numRowsPerQueryPart" json:"num_rows_per_query_part,omitempty"`
	2272	Algorithm SplitQueryRequest_Algorithm `protobuf:"varint,9,opt,name=algorithm,enum=query.SplitQueryRequest_Algorithm" json:"algorithm,omitempty"`
	2273	}
	2274
	2275	func (m SplitQueryRequest) Reset() { m = SplitQueryRequest{} }
	2276	func (m *SplitQueryRequest) String() string { return "TODO" }
	2277
	2278
	2279
	2280	func (m SplitQueryRequest) GetEffectiveCallerId() vtrpc.CallerID {
	2281	if m != nil {
	2282	return m.EffectiveCallerId
	2283	}
	2284	return nil
	2285	}
	2286
	2287	func (m SplitQueryRequest) GetImmediateCallerId() VTGateCallerID {
	2288	if m != nil {
	2289	return m.ImmediateCallerId
	2290	}
	2291	return nil
	2292	}
	2293
	2294	func (m SplitQueryRequest) GetTarget() Target {
	2295	if m != nil {
	2296	return m.Target
	2297	}
	2298	return nil
	2299	}
	2300
	2301	func (m SplitQueryRequest) GetQuery() BoundQuery {
	2302	if m != nil {
	2303	return m.Query
	2304	}
	2305	return nil
	2306	}
	2307
	2308	func (m *SplitQueryRequest) GetSplitColumn() []string {
	2309	if m != nil {
	2310	return m.SplitColumn
	2311	}
	2312	return nil
	2313	}
	2314
	2315	func (m *SplitQueryRequest) GetSplitCount() int64 {
	2316	if m != nil {
	2317	return m.SplitCount
	2318	}
	2319	return 0
	2320	}
	2321
	2322	func (m *SplitQueryRequest) GetNumRowsPerQueryPart() int64 {
	2323	if m != nil {
	2324	return m.NumRowsPerQueryPart
	2325	}
	2326	return 0
	2327	}
	2328
	2329	func (m *SplitQueryRequest) GetAlgorithm() SplitQueryRequest_Algorithm {
	2330	if m != nil {
	2331	return m.Algorithm
	2332	}
	2333	return SplitQueryRequest_EQUAL_SPLITS
	2334	}
	2335
	2336	// QuerySplit represents one query to execute on the tablet
	2337	type QuerySplit struct {
	2338	// query is the query to execute
	2339	Query *BoundQuery `protobuf:"bytes,1,opt,name=query" json:"query,omitempty"`
	2340	// row_count is the approximate row count the query will return
	2341	RowCount int64 `protobuf:"varint,2,opt,name=row_count,json=rowCount" json:"row_count,omitempty"`
	2342	}
	2343
	2344	func (m QuerySplit) Reset() { m = QuerySplit{} }
	2345	func (m *QuerySplit) String() string { return "TODO" }
	2346
	2347
	2348
	2349	func (m QuerySplit) GetQuery() BoundQuery {
	2350	if m != nil {
	2351	return m.Query
	2352	}
	2353	return nil
	2354	}
	2355
	2356	func (m *QuerySplit) GetRowCount() int64 {
	2357	if m != nil {
	2358	return m.RowCount
	2359	}
	2360	return 0
	2361	}
	2362
	2363	// SplitQueryResponse is returned by SplitQuery and represents all the queries
	2364	// to execute in order to get the entire data set.
	2365	type SplitQueryResponse struct {
	2366	Queries []*QuerySplit `protobuf:"bytes,1,rep,name=queries" json:"queries,omitempty"`
	2367	}
	2368
	2369	func (m SplitQueryResponse) Reset() { m = SplitQueryResponse{} }
	2370	func (m *SplitQueryResponse) String() string { return "TODO" }
	2371
	2372
	2373
	2374	func (m SplitQueryResponse) GetQueries() []QuerySplit {
	2375	if m != nil {
	2376	return m.Queries
	2377	}
	2378	return nil
	2379	}
	2380
	2381	// StreamHealthRequest is the payload for StreamHealth
	2382	type StreamHealthRequest struct {
	2383	}
	2384
	2385	func (m StreamHealthRequest) Reset() { m = StreamHealthRequest{} }
	2386	func (m *StreamHealthRequest) String() string { return "TODO" }
	2387
	2388
	2389
	2390	// RealtimeStats contains information about the tablet status
	2391	type RealtimeStats struct {
	2392	// health_error is the last error we got from health check,
	2393	// or empty is the server is healthy. This is used for subset selection,
	2394	// we do not send queries to servers that are not healthy.
	2395	HealthError string `protobuf:"bytes,1,opt,name=health_error,json=healthError" json:"health_error,omitempty"`
	2396	// seconds_behind_master is populated for slaves only. It indicates
	2397	// how far behind on (MySQL) replication a slave currently is. It is used
	2398	// by clients for subset selection (so we don't try to send traffic
	2399	// to tablets that are too far behind).
	2400	// NOTE: This field must not be evaluated if "health_error" is not empty.
	2401	// TODO(mberlin): Let's switch it to int64 instead?
	2402	SecondsBehindMaster uint32 `protobuf:"varint,2,opt,name=seconds_behind_master,json=secondsBehindMaster" json:"seconds_behind_master,omitempty"`
	2403	// bin_log_players_count is the number of currently running binlog players.
	2404	// if the value is 0, it means that filtered replication is currently not
	2405	// running on the tablet. If >0, filtered replication is running.
	2406	// NOTE: This field must not be evaluated if "health_error" is not empty.
	2407	BinlogPlayersCount int32 `protobuf:"varint,3,opt,name=binlog_players_count,json=binlogPlayersCount" json:"binlog_players_count,omitempty"`
	2408	// seconds_behind_master_filtered_replication is populated for the receiving
	2409	// master of an ongoing filtered replication only.
	2410	// It specifies how far the receiving master lags behind the sending master.
	2411	// NOTE: This field must not be evaluated if "health_error" is not empty.
	2412	// NOTE: This field must not be evaluated if "bin_log_players_count" is 0.
	2413	SecondsBehindMasterFilteredReplication int64 `protobuf:"varint,4,opt,name=seconds_behind_master_filtered_replication,json=secondsBehindMasterFilteredReplication" json:"seconds_behind_master_filtered_replication,omitempty"`
	2414	// cpu_usage is used for load-based balancing
	2415	CpuUsage float64 `protobuf:"fixed64,5,opt,name=cpu_usage,json=cpuUsage" json:"cpu_usage,omitempty"`
	2416	// qps is the average QPS (queries per second) rate in the last XX seconds
	2417	// where XX is usually 60 (See query_service_stats.go).
	2418	Qps float64 `protobuf:"fixed64,6,opt,name=qps" json:"qps,omitempty"`
	2419	}
	2420
	2421	func (m RealtimeStats) Reset() { m = RealtimeStats{} }
	2422	func (m *RealtimeStats) String() string { return "TODO" }
	2423
	2424
	2425
	2426	func (m *RealtimeStats) GetHealthError() string {
	2427	if m != nil {
	2428	return m.HealthError
	2429	}
	2430	return ""
	2431	}
	2432
	2433	func (m *RealtimeStats) GetSecondsBehindMaster() uint32 {
	2434	if m != nil {
	2435	return m.SecondsBehindMaster
	2436	}
	2437	return 0
	2438	}
	2439
	2440	func (m *RealtimeStats) GetBinlogPlayersCount() int32 {
	2441	if m != nil {
	2442	return m.BinlogPlayersCount
	2443	}
	2444	return 0
	2445	}
	2446
	2447	func (m *RealtimeStats) GetSecondsBehindMasterFilteredReplication() int64 {
	2448	if m != nil {
	2449	return m.SecondsBehindMasterFilteredReplication
	2450	}
	2451	return 0
	2452	}
	2453
	2454	func (m *RealtimeStats) GetCpuUsage() float64 {
	2455	if m != nil {
	2456	return m.CpuUsage
	2457	}
	2458	return 0
	2459	}
	2460
	2461	func (m *RealtimeStats) GetQps() float64 {
	2462	if m != nil {
	2463	return m.Qps
	2464	}
	2465	return 0
	2466	}
	2467
	2468	// StreamHealthResponse is streamed by StreamHealth on a regular basis
	2469	type StreamHealthResponse struct {
	2470	// target is the current server type. Only queries with that exact Target
	2471	// record will be accepted.
	2472	Target *Target `protobuf:"bytes,1,opt,name=target" json:"target,omitempty"`
	2473	// serving is true iff the tablet is serving. A tablet may not be serving
	2474	// if filtered replication is enabled on a master for instance,
	2475	// or if a replica should not be used because the keyspace is being resharded.
	2476	Serving bool `protobuf:"varint,2,opt,name=serving" json:"serving,omitempty"`
	2477	// tablet_externally_reparented_timestamp can be interpreted as the last time
	2478	// we knew that this tablet was the MASTER of this shard.
	2479	//
	2480	// It is used by vtgate when determining the current MASTER of a shard.
	2481	// If vtgate sees more than one MASTER tablet, this timestamp is used
	2482	// as tiebreaker where the MASTER with the highest timestamp wins.
	2483	// Another usage of this timestamp is in go/vt/vtgate/buffer to detect the end
	2484	// of a reparent (failover) and stop buffering.
	2485	//
	2486	// In practice, this field is set to:
	2487	// a) the last time the RPC tabletmanager.TabletExternallyReparented was
	2488	// called on this tablet (usually done by an external failover tool e.g.
	2489	// Orchestrator). The failover tool can call this as long as we are the
	2490	// master i.e. even ages after the last reparent occurred.
	2491	// OR
	2492	// b) the last time an active reparent was executed through a vtctl command
	2493	// (InitShardMaster, PlannedReparentShard, EmergencyReparentShard)
	2494	// OR
	2495	// c) the last time vttablet was started and it initialized its tablet type
	2496	// as MASTER because it was recorded as the shard's current master in the
	2497	// topology (see go/vt/vttablet/tabletmanager/init_tablet.go)
	2498	// OR
	2499	// d) 0 if the vttablet was never a MASTER.
	2500	TabletExternallyReparentedTimestamp int64 `protobuf:"varint,3,opt,name=tablet_externally_reparented_timestamp,json=tabletExternallyReparentedTimestamp" json:"tablet_externally_reparented_timestamp,omitempty"`
	2501	// realtime_stats contains information about the tablet status
	2502	RealtimeStats *RealtimeStats `protobuf:"bytes,4,opt,name=realtime_stats,json=realtimeStats" json:"realtime_stats,omitempty"`
	2503	// tablet_alias is the alias of the sending tablet. The discovery/healthcheck.go
	2504	// code uses it to verify that it's talking to the correct tablet and that it
	2505	// hasn't changed in the meantime e.g. due to tablet restarts where ports or
	2506	// ips have been reused but assigned differently.
	2507	TabletAlias *topodata.TabletAlias `protobuf:"bytes,5,opt,name=tablet_alias,json=tabletAlias" json:"tablet_alias,omitempty"`
	2508	}
	2509
	2510	func (m StreamHealthResponse) Reset() { m = StreamHealthResponse{} }
	2511	func (m *StreamHealthResponse) String() string { return "TODO" }
	2512
	2513
	2514
	2515	func (m StreamHealthResponse) GetTarget() Target {
	2516	if m != nil {
	2517	return m.Target
	2518	}
	2519	return nil
	2520	}
	2521
	2522	func (m *StreamHealthResponse) GetServing() bool {
	2523	if m != nil {
	2524	return m.Serving
	2525	}
	2526	return false
	2527	}
	2528
	2529	func (m *StreamHealthResponse) GetTabletExternallyReparentedTimestamp() int64 {
	2530	if m != nil {
	2531	return m.TabletExternallyReparentedTimestamp
	2532	}
	2533	return 0
	2534	}
	2535
	2536	func (m StreamHealthResponse) GetRealtimeStats() RealtimeStats {
	2537	if m != nil {
	2538	return m.RealtimeStats
	2539	}
	2540	return nil
	2541	}
	2542
	2543	func (m StreamHealthResponse) GetTabletAlias() topodata.TabletAlias {
	2544	if m != nil {
	2545	return m.TabletAlias
	2546	}
	2547	return nil
	2548	}
	2549
	2550	// UpdateStreamRequest is the payload for UpdateStream. At most one of
	2551	// position and timestamp can be set. If neither is set, we will start
	2552	// streaming from the current binlog position.
	2553	type UpdateStreamRequest struct {
	2554	EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
	2555	ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
	2556	Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
	2557	// If position is set, we will start the streaming from that replication
	2558	// position. Incompatible with timestamp.
	2559	Position string `protobuf:"bytes,4,opt,name=position" json:"position,omitempty"`
	2560	// If timestamp is set, we will start the streaming from the first
	2561	// event in the binlogs that have that timestamp. Incompatible with position.
	2562	Timestamp int64 `protobuf:"varint,5,opt,name=timestamp" json:"timestamp,omitempty"`
	2563	}
	2564
	2565	func (m UpdateStreamRequest) Reset() { m = UpdateStreamRequest{} }
	2566	func (m *UpdateStreamRequest) String() string { return "TODO" }
	2567
	2568
	2569
	2570	func (m UpdateStreamRequest) GetEffectiveCallerId() vtrpc.CallerID {
	2571	if m != nil {
	2572	return m.EffectiveCallerId
	2573	}
	2574	return nil
	2575	}
	2576
	2577	func (m UpdateStreamRequest) GetImmediateCallerId() VTGateCallerID {
	2578	if m != nil {
	2579	return m.ImmediateCallerId
	2580	}
	2581	return nil
	2582	}
	2583
	2584	func (m UpdateStreamRequest) GetTarget() Target {
	2585	if m != nil {
	2586	return m.Target
	2587	}
	2588	return nil
	2589	}
	2590
	2591	func (m *UpdateStreamRequest) GetPosition() string {
	2592	if m != nil {
	2593	return m.Position
	2594	}
	2595	return ""
	2596	}
	2597
	2598	func (m *UpdateStreamRequest) GetTimestamp() int64 {
	2599	if m != nil {
	2600	return m.Timestamp
	2601	}
	2602	return 0
	2603	}
	2604
	2605	// UpdateStreamResponse is returned by UpdateStream
	2606	type UpdateStreamResponse struct {
	2607	Event *StreamEvent `protobuf:"bytes,1,opt,name=event" json:"event,omitempty"`
	2608	}
	2609
	2610	func (m UpdateStreamResponse) Reset() { m = UpdateStreamResponse{} }
	2611	func (m *UpdateStreamResponse) String() string { return "TODO" }
	2612
	2613
	2614
	2615	func (m UpdateStreamResponse) GetEvent() StreamEvent {
	2616	if m != nil {
	2617	return m.Event
	2618	}
	2619	return nil
	2620	}
	2621
	2622	// TransactionMetadata contains the metadata for a distributed transaction.
	2623	type TransactionMetadata struct {
	2624	Dtid string `protobuf:"bytes,1,opt,name=dtid" json:"dtid,omitempty"`
	2625	State TransactionState `protobuf:"varint,2,opt,name=state,enum=query.TransactionState" json:"state,omitempty"`
	2626	TimeCreated int64 `protobuf:"varint,3,opt,name=time_created,json=timeCreated" json:"time_created,omitempty"`
	2627	Participants []*Target `protobuf:"bytes,4,rep,name=participants" json:"participants,omitempty"`
	2628	}
	2629
	2630	func (m TransactionMetadata) Reset() { m = TransactionMetadata{} }
	2631	func (m *TransactionMetadata) String() string { return "TODO" }
	2632
	2633
	2634
	2635	func (m *TransactionMetadata) GetDtid() string {
	2636	if m != nil {
	2637	return m.Dtid
	2638	}
	2639	return ""
	2640	}
	2641
	2642	func (m *TransactionMetadata) GetState() TransactionState {
	2643	if m != nil {
	2644	return m.State
	2645	}
	2646	return TransactionState_UNKNOWN
	2647	}
	2648
	2649	func (m *TransactionMetadata) GetTimeCreated() int64 {
	2650	if m != nil {
	2651	return m.TimeCreated
	2652	}
	2653	return 0
	2654	}
	2655
	2656	func (m TransactionMetadata) GetParticipants() []Target {
	2657	if m != nil {
	2658	return m.Participants
	2659	}
	2660	return nil
	2661	}
	2662
	2663	func init() {
	2664	proto.RegisterType((*Target)(nil), "query.Target")
	2665	proto.RegisterType((*VTGateCallerID)(nil), "query.VTGateCallerID")
	2666	proto.RegisterType((*EventToken)(nil), "query.EventToken")
	2667	proto.RegisterType((*Value)(nil), "query.Value")
	2668	proto.RegisterType((*BindVariable)(nil), "query.BindVariable")
	2669	proto.RegisterType((*BoundQuery)(nil), "query.BoundQuery")
	2670	proto.RegisterType((*ExecuteOptions)(nil), "query.ExecuteOptions")
	2671	proto.RegisterType((*Field)(nil), "query.Field")
	2672	proto.RegisterType((*Row)(nil), "query.Row")
	2673	proto.RegisterType((*ResultExtras)(nil), "query.ResultExtras")
	2674	proto.RegisterType((*QueryResult)(nil), "query.QueryResult")
	2675	proto.RegisterType((*StreamEvent)(nil), "query.StreamEvent")
	2676	proto.RegisterType((*StreamEvent_Statement)(nil), "query.StreamEvent.Statement")
	2677	proto.RegisterType((*ExecuteRequest)(nil), "query.ExecuteRequest")
	2678	proto.RegisterType((*ExecuteResponse)(nil), "query.ExecuteResponse")
	2679	proto.RegisterType((*ResultWithError)(nil), "query.ResultWithError")
	2680	proto.RegisterType((*ExecuteBatchRequest)(nil), "query.ExecuteBatchRequest")
	2681	proto.RegisterType((*ExecuteBatchResponse)(nil), "query.ExecuteBatchResponse")
	2682	proto.RegisterType((*StreamExecuteRequest)(nil), "query.StreamExecuteRequest")
	2683	proto.RegisterType((*StreamExecuteResponse)(nil), "query.StreamExecuteResponse")
	2684	proto.RegisterType((*BeginRequest)(nil), "query.BeginRequest")
	2685	proto.RegisterType((*BeginResponse)(nil), "query.BeginResponse")
	2686	proto.RegisterType((*CommitRequest)(nil), "query.CommitRequest")
	2687	proto.RegisterType((*CommitResponse)(nil), "query.CommitResponse")
	2688	proto.RegisterType((*RollbackRequest)(nil), "query.RollbackRequest")
	2689	proto.RegisterType((*RollbackResponse)(nil), "query.RollbackResponse")
	2690	proto.RegisterType((*PrepareRequest)(nil), "query.PrepareRequest")
	2691	proto.RegisterType((*PrepareResponse)(nil), "query.PrepareResponse")
	2692	proto.RegisterType((*CommitPreparedRequest)(nil), "query.CommitPreparedRequest")
	2693	proto.RegisterType((*CommitPreparedResponse)(nil), "query.CommitPreparedResponse")
	2694	proto.RegisterType((*RollbackPreparedRequest)(nil), "query.RollbackPreparedRequest")
	2695	proto.RegisterType((*RollbackPreparedResponse)(nil), "query.RollbackPreparedResponse")
	2696	proto.RegisterType((*CreateTransactionRequest)(nil), "query.CreateTransactionRequest")
	2697	proto.RegisterType((*CreateTransactionResponse)(nil), "query.CreateTransactionResponse")
	2698	proto.RegisterType((*StartCommitRequest)(nil), "query.StartCommitRequest")
	2699	proto.RegisterType((*StartCommitResponse)(nil), "query.StartCommitResponse")
	2700	proto.RegisterType((*SetRollbackRequest)(nil), "query.SetRollbackRequest")
	2701	proto.RegisterType((*SetRollbackResponse)(nil), "query.SetRollbackResponse")
	2702	proto.RegisterType((*ConcludeTransactionRequest)(nil), "query.ConcludeTransactionRequest")
	2703	proto.RegisterType((*ConcludeTransactionResponse)(nil), "query.ConcludeTransactionResponse")
	2704	proto.RegisterType((*ReadTransactionRequest)(nil), "query.ReadTransactionRequest")
	2705	proto.RegisterType((*ReadTransactionResponse)(nil), "query.ReadTransactionResponse")
	2706	proto.RegisterType((*BeginExecuteRequest)(nil), "query.BeginExecuteRequest")
	2707	proto.RegisterType((*BeginExecuteResponse)(nil), "query.BeginExecuteResponse")
	2708	proto.RegisterType((*BeginExecuteBatchRequest)(nil), "query.BeginExecuteBatchRequest")
	2709	proto.RegisterType((*BeginExecuteBatchResponse)(nil), "query.BeginExecuteBatchResponse")
	2710	proto.RegisterType((*MessageStreamRequest)(nil), "query.MessageStreamRequest")
	2711	proto.RegisterType((*MessageStreamResponse)(nil), "query.MessageStreamResponse")
	2712	proto.RegisterType((*MessageAckRequest)(nil), "query.MessageAckRequest")
	2713	proto.RegisterType((*MessageAckResponse)(nil), "query.MessageAckResponse")
	2714	proto.RegisterType((*SplitQueryRequest)(nil), "query.SplitQueryRequest")
	2715	proto.RegisterType((*QuerySplit)(nil), "query.QuerySplit")
	2716	proto.RegisterType((*SplitQueryResponse)(nil), "query.SplitQueryResponse")
	2717	proto.RegisterType((*StreamHealthRequest)(nil), "query.StreamHealthRequest")
	2718	proto.RegisterType((*RealtimeStats)(nil), "query.RealtimeStats")
	2719	proto.RegisterType((*StreamHealthResponse)(nil), "query.StreamHealthResponse")
	2720	proto.RegisterType((*UpdateStreamRequest)(nil), "query.UpdateStreamRequest")
	2721	proto.RegisterType((*UpdateStreamResponse)(nil), "query.UpdateStreamResponse")
	2722	proto.RegisterType((*TransactionMetadata)(nil), "query.TransactionMetadata")
	2723	proto.RegisterEnum("query.MySqlFlag", MySqlFlag_name, MySqlFlag_value)
	2724	proto.RegisterEnum("query.Flag", Flag_name, Flag_value)
	2725	proto.RegisterEnum("query.Type", Type_name, Type_value)
	2726	proto.RegisterEnum("query.TransactionState", TransactionState_name, TransactionState_value)
	2727	proto.RegisterEnum("query.ExecuteOptions_IncludedFields", ExecuteOptions_IncludedFields_name, ExecuteOptions_IncludedFields_value)
	2728	proto.RegisterEnum("query.ExecuteOptions_Workload", ExecuteOptions_Workload_name, ExecuteOptions_Workload_value)
	2729	proto.RegisterEnum("query.ExecuteOptions_TransactionIsolation", ExecuteOptions_TransactionIsolation_name, ExecuteOptions_TransactionIsolation_value)
	2730	proto.RegisterEnum("query.StreamEvent_Statement_Category", StreamEvent_Statement_Category_name, StreamEvent_Statement_Category_value)
	2731	proto.RegisterEnum("query.SplitQueryRequest_Algorithm", SplitQueryRequest_Algorithm_name, SplitQueryRequest_Algorithm_value)
	2732	}
	2733
	2734	*/

vendor/github.com/xwb1989/sqlparser/dependency/sqltypes/bind_variables.go+266−0

@@ -0,0 +1,266 @@
	1	/*
	2	Copyright 2017 Google Inc.
	3
	4	Licensed under the Apache License, Version 2.0 (the "License");
	5	you may not use this file except in compliance with the License.
	6	You may obtain a copy of the License at
	7
	8	http://www.apache.org/licenses/LICENSE-2.0
	9
	10	Unless required by applicable law or agreed to in writing, software
	11	distributed under the License is distributed on an "AS IS" BASIS,
	12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	See the License for the specific language governing permissions and
	14	limitations under the License.
	15	*/
	16
	17	package sqltypes
	18
	19	import (
	20	"errors"
	21	"fmt"
	22	"reflect"
	23	"strconv"
	24
	25	"github.com/xwb1989/sqlparser/dependency/querypb"
	26	)
	27
	28	// NullBindVariable is a bindvar with NULL value.
	29	var NullBindVariable = &querypb.BindVariable{Type: querypb.Type_NULL_TYPE}
	30
	31	// ValueToProto converts Value to a *querypb.Value.
	32	func ValueToProto(v Value) *querypb.Value {
	33	return &querypb.Value{Type: v.typ, Value: v.val}
	34	}
	35
	36	// ProtoToValue converts a *querypb.Value to a Value.
	37	func ProtoToValue(v *querypb.Value) Value {
	38	return MakeTrusted(v.Type, v.Value)
	39	}
	40
	41	// BuildBindVariables builds a map[string]*querypb.BindVariable from a map[string]interface{}.
	42	func BuildBindVariables(in map[string]interface{}) (map[string]*querypb.BindVariable, error) {
	43	if len(in) == 0 {
	44	return nil, nil
	45	}
	46
	47	out := make(map[string]*querypb.BindVariable, len(in))
	48	for k, v := range in {
	49	bv, err := BuildBindVariable(v)
	50	if err != nil {
	51	return nil, fmt.Errorf("%s: %v", k, err)
	52	}
	53	out[k] = bv
	54	}
	55	return out, nil
	56	}
	57
	58	// Int32BindVariable converts an int32 to a bind var.
	59	func Int32BindVariable(v int32) *querypb.BindVariable {
	60	return ValueBindVariable(NewInt32(v))
	61	}
	62
	63	// Int64BindVariable converts an int64 to a bind var.
	64	func Int64BindVariable(v int64) *querypb.BindVariable {
	65	return ValueBindVariable(NewInt64(v))
	66	}
	67
	68	// Uint64BindVariable converts a uint64 to a bind var.
	69	func Uint64BindVariable(v uint64) *querypb.BindVariable {
	70	return ValueBindVariable(NewUint64(v))
	71	}
	72
	73	// Float64BindVariable converts a float64 to a bind var.
	74	func Float64BindVariable(v float64) *querypb.BindVariable {
	75	return ValueBindVariable(NewFloat64(v))
	76	}
	77
	78	// StringBindVariable converts a string to a bind var.
	79	func StringBindVariable(v string) *querypb.BindVariable {
	80	return ValueBindVariable(NewVarChar(v))
	81	}
	82
	83	// BytesBindVariable converts a []byte to a bind var.
	84	func BytesBindVariable(v []byte) *querypb.BindVariable {
	85	return &querypb.BindVariable{Type: VarBinary, Value: v}
	86	}
	87
	88	// ValueBindVariable converts a Value to a bind var.
	89	func ValueBindVariable(v Value) *querypb.BindVariable {
	90	return &querypb.BindVariable{Type: v.typ, Value: v.val}
	91	}
	92
	93	// BuildBindVariable builds a *querypb.BindVariable from a valid input type.
	94	func BuildBindVariable(v interface{}) (*querypb.BindVariable, error) {
	95	switch v := v.(type) {
	96	case string:
	97	return StringBindVariable(v), nil
	98	case []byte:
	99	return BytesBindVariable(v), nil
	100	case int:
	101	return &querypb.BindVariable{
	102	Type: querypb.Type_INT64,
	103	Value: strconv.AppendInt(nil, int64(v), 10),
	104	}, nil
	105	case int64:
	106	return Int64BindVariable(v), nil
	107	case uint64:
	108	return Uint64BindVariable(v), nil
	109	case float64:
	110	return Float64BindVariable(v), nil
	111	case nil:
	112	return NullBindVariable, nil
	113	case Value:
	114	return ValueBindVariable(v), nil
	115	case *querypb.BindVariable:
	116	return v, nil
	117	case []interface{}:
	118	bv := &querypb.BindVariable{
	119	Type: querypb.Type_TUPLE,
	120	Values: make([]*querypb.Value, len(v)),
	121	}
	122	values := make([]querypb.Value, len(v))
	123	for i, lv := range v {
	124	lbv, err := BuildBindVariable(lv)
	125	if err != nil {
	126	return nil, err
	127	}
	128	values[i].Type = lbv.Type
	129	values[i].Value = lbv.Value
	130	bv.Values[i] = &values[i]
	131	}
	132	return bv, nil
	133	case []string:
	134	bv := &querypb.BindVariable{
	135	Type: querypb.Type_TUPLE,
	136	Values: make([]*querypb.Value, len(v)),
	137	}
	138	values := make([]querypb.Value, len(v))
	139	for i, lv := range v {
	140	values[i].Type = querypb.Type_VARCHAR
	141	values[i].Value = []byte(lv)
	142	bv.Values[i] = &values[i]
	143	}
	144	return bv, nil
	145	case [][]byte:
	146	bv := &querypb.BindVariable{
	147	Type: querypb.Type_TUPLE,
	148	Values: make([]*querypb.Value, len(v)),
	149	}
	150	values := make([]querypb.Value, len(v))
	151	for i, lv := range v {
	152	values[i].Type = querypb.Type_VARBINARY
	153	values[i].Value = lv
	154	bv.Values[i] = &values[i]
	155	}
	156	return bv, nil
	157	case []int:
	158	bv := &querypb.BindVariable{
	159	Type: querypb.Type_TUPLE,
	160	Values: make([]*querypb.Value, len(v)),
	161	}
	162	values := make([]querypb.Value, len(v))
	163	for i, lv := range v {
	164	values[i].Type = querypb.Type_INT64
	165	values[i].Value = strconv.AppendInt(nil, int64(lv), 10)
	166	bv.Values[i] = &values[i]
	167	}
	168	return bv, nil
	169	case []int64:
	170	bv := &querypb.BindVariable{
	171	Type: querypb.Type_TUPLE,
	172	Values: make([]*querypb.Value, len(v)),
	173	}
	174	values := make([]querypb.Value, len(v))
	175	for i, lv := range v {
	176	values[i].Type = querypb.Type_INT64
	177	values[i].Value = strconv.AppendInt(nil, lv, 10)
	178	bv.Values[i] = &values[i]
	179	}
	180	return bv, nil
	181	case []uint64:
	182	bv := &querypb.BindVariable{
	183	Type: querypb.Type_TUPLE,
	184	Values: make([]*querypb.Value, len(v)),
	185	}
	186	values := make([]querypb.Value, len(v))
	187	for i, lv := range v {
	188	values[i].Type = querypb.Type_UINT64
	189	values[i].Value = strconv.AppendUint(nil, lv, 10)
	190	bv.Values[i] = &values[i]
	191	}
	192	return bv, nil
	193	case []float64:
	194	bv := &querypb.BindVariable{
	195	Type: querypb.Type_TUPLE,
	196	Values: make([]*querypb.Value, len(v)),
	197	}
	198	values := make([]querypb.Value, len(v))
	199	for i, lv := range v {
	200	values[i].Type = querypb.Type_FLOAT64
	201	values[i].Value = strconv.AppendFloat(nil, lv, 'g', -1, 64)
	202	bv.Values[i] = &values[i]
	203	}
	204	return bv, nil
	205	}
	206	return nil, fmt.Errorf("type %T not supported as bind var: %v", v, v)
	207	}
	208
	209	// ValidateBindVariables validates a map[string]*querypb.BindVariable.
	210	func ValidateBindVariables(bv map[string]*querypb.BindVariable) error {
	211	for k, v := range bv {
	212	if err := ValidateBindVariable(v); err != nil {
	213	return fmt.Errorf("%s: %v", k, err)
	214	}
	215	}
	216	return nil
	217	}
	218
	219	// ValidateBindVariable returns an error if the bind variable has inconsistent
	220	// fields.
	221	func ValidateBindVariable(bv *querypb.BindVariable) error {
	222	if bv == nil {
	223	return errors.New("bind variable is nil")
	224	}
	225
	226	if bv.Type == querypb.Type_TUPLE {
	227	if len(bv.Values) == 0 {
	228	return errors.New("empty tuple is not allowed")
	229	}
	230	for _, val := range bv.Values {
	231	if val.Type == querypb.Type_TUPLE {
	232	return errors.New("tuple not allowed inside another tuple")
	233	}
	234	if err := ValidateBindVariable(&querypb.BindVariable{Type: val.Type, Value: val.Value}); err != nil {
	235	return err
	236	}
	237	}
	238	return nil
	239	}
	240
	241	// If NewValue succeeds, the value is valid.
	242	_, err := NewValue(bv.Type, bv.Value)
	243	return err
	244	}
	245
	246	// BindVariableToValue converts a bind var into a Value.
	247	func BindVariableToValue(bv *querypb.BindVariable) (Value, error) {
	248	if bv.Type == querypb.Type_TUPLE {
	249	return NULL, errors.New("cannot convert a TUPLE bind var into a value")
	250	}
	251	return MakeTrusted(bv.Type, bv.Value), nil
	252	}
	253
	254	// BindVariablesEqual compares two maps of bind variables.
	255	func BindVariablesEqual(x, y map[string]*querypb.BindVariable) bool {
	256	return reflect.DeepEqual(&querypb.BoundQuery{BindVariables: x}, &querypb.BoundQuery{BindVariables: y})
	257	}
	258
	259	// CopyBindVariables returns a shallow-copy of the given bindVariables map.
	260	func CopyBindVariables(bindVariables map[string]querypb.BindVariable) map[string]querypb.BindVariable {
	261	result := make(map[string]*querypb.BindVariable, len(bindVariables))
	262	for key, value := range bindVariables {
	263	result[key] = value
	264	}
	265	return result
	266	}

vendor/github.com/xwb1989/sqlparser/dependency/sqltypes/plan_value.go+259−0

@@ -0,0 +1,259 @@
	1	/*
	2	Copyright 2017 Google Inc.
	3
	4	Licensed under the Apache License, Version 2.0 (the "License");
	5	you may not use this file except in compliance with the License.
	6	You may obtain a copy of the License at
	7
	8	http://www.apache.org/licenses/LICENSE-2.0
	9
	10	Unless required by applicable law or agreed to in writing, software
	11	distributed under the License is distributed on an "AS IS" BASIS,
	12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	See the License for the specific language governing permissions and
	14	limitations under the License.
	15	*/
	16
	17	package sqltypes
	18
	19	import (
	20	"encoding/json"
	21	"errors"
	22	"fmt"
	23
	24	"github.com/xwb1989/sqlparser/dependency/querypb"
	25	)
	26
	27	// PlanValue represents a value or a list of values for
	28	// a column that will later be resolved using bind vars and used
	29	// to perform plan actions like generating the final query or
	30	// deciding on a route.
	31	//
	32	// Plan values are typically used as a slice ([]planValue)
	33	// where each entry is for one column. For situations where
	34	// the required output is a list of rows (like in the case
	35	// of multi-value inserts), the representation is pivoted.
	36	// For example, a statement like this:
	37	// INSERT INTO t VALUES (1, 2), (3, 4)
	38	// will be represented as follows:
	39	// []PlanValue{
	40	// Values: {1, 3},
	41	// Values: {2, 4},
	42	// }
	43	//
	44	// For WHERE clause items that contain a combination of
	45	// equality expressions and IN clauses like this:
	46	// WHERE pk1 = 1 AND pk2 IN (2, 3, 4)
	47	// The plan values will be represented as follows:
	48	// []PlanValue{
	49	// Value: 1,
	50	// Values: {2, 3, 4},
	51	// }
	52	// When converted into rows, columns with single values
	53	// are replicated as the same for all rows:
	54	// [][]Value{
	55	// {1, 2},
	56	// {1, 3},
	57	// {1, 4},
	58	// }
	59	type PlanValue struct {
	60	Key string
	61	Value Value
	62	ListKey string
	63	Values []PlanValue
	64	}
	65
	66	// IsNull returns true if the PlanValue is NULL.
	67	func (pv PlanValue) IsNull() bool {
	68	return pv.Key == "" && pv.Value.IsNull() && pv.ListKey == "" && pv.Values == nil
	69	}
	70
	71	// IsList returns true if the PlanValue is a list.
	72	func (pv PlanValue) IsList() bool {
	73	return pv.ListKey != "" \|\| pv.Values != nil
	74	}
	75
	76	// ResolveValue resolves a PlanValue as a single value based on the supplied bindvars.
	77	func (pv PlanValue) ResolveValue(bindVars map[string]*querypb.BindVariable) (Value, error) {
	78	switch {
	79	case pv.Key != "":
	80	bv, err := pv.lookupValue(bindVars)
	81	if err != nil {
	82	return NULL, err
	83	}
	84	return MakeTrusted(bv.Type, bv.Value), nil
	85	case !pv.Value.IsNull():
	86	return pv.Value, nil
	87	case pv.ListKey != "" \|\| pv.Values != nil:
	88	// This code is unreachable because the parser does not allow
	89	// multi-value constructs where a single value is expected.
	90	return NULL, errors.New("a list was supplied where a single value was expected")
	91	}
	92	return NULL, nil
	93	}
	94
	95	func (pv PlanValue) lookupValue(bindVars map[string]querypb.BindVariable) (querypb.BindVariable, error) {
	96	bv, ok := bindVars[pv.Key]
	97	if !ok {
	98	return nil, fmt.Errorf("missing bind var %s", pv.Key)
	99	}
	100	if bv.Type == querypb.Type_TUPLE {
	101	return nil, fmt.Errorf("TUPLE was supplied for single value bind var %s", pv.ListKey)
	102	}
	103	return bv, nil
	104	}
	105
	106	// ResolveList resolves a PlanValue as a list of values based on the supplied bindvars.
	107	func (pv PlanValue) ResolveList(bindVars map[string]*querypb.BindVariable) ([]Value, error) {
	108	switch {
	109	case pv.ListKey != "":
	110	bv, err := pv.lookupList(bindVars)
	111	if err != nil {
	112	return nil, err
	113	}
	114	values := make([]Value, 0, len(bv.Values))
	115	for _, val := range bv.Values {
	116	values = append(values, MakeTrusted(val.Type, val.Value))
	117	}
	118	return values, nil
	119	case pv.Values != nil:
	120	values := make([]Value, 0, len(pv.Values))
	121	for _, val := range pv.Values {
	122	v, err := val.ResolveValue(bindVars)
	123	if err != nil {
	124	return nil, err
	125	}
	126	values = append(values, v)
	127	}
	128	return values, nil
	129	}
	130	// This code is unreachable because the parser does not allow
	131	// single value constructs where multiple values are expected.
	132	return nil, errors.New("a single value was supplied where a list was expected")
	133	}
	134
	135	func (pv PlanValue) lookupList(bindVars map[string]querypb.BindVariable) (querypb.BindVariable, error) {
	136	bv, ok := bindVars[pv.ListKey]
	137	if !ok {
	138	return nil, fmt.Errorf("missing bind var %s", pv.ListKey)
	139	}
	140	if bv.Type != querypb.Type_TUPLE {
	141	return nil, fmt.Errorf("single value was supplied for TUPLE bind var %s", pv.ListKey)
	142	}
	143	return bv, nil
	144	}
	145
	146	// MarshalJSON should be used only for testing.
	147	func (pv PlanValue) MarshalJSON() ([]byte, error) {
	148	switch {
	149	case pv.Key != "":
	150	return json.Marshal(":" + pv.Key)
	151	case !pv.Value.IsNull():
	152	if pv.Value.IsIntegral() {
	153	return pv.Value.ToBytes(), nil
	154	}
	155	return json.Marshal(pv.Value.ToString())
	156	case pv.ListKey != "":
	157	return json.Marshal("::" + pv.ListKey)
	158	case pv.Values != nil:
	159	return json.Marshal(pv.Values)
	160	}
	161	return []byte("null"), nil
	162	}
	163
	164	func rowCount(pvs []PlanValue, bindVars map[string]*querypb.BindVariable) (int, error) {
	165	count := -1
	166	setCount := func(l int) error {
	167	switch count {
	168	case -1:
	169	count = l
	170	return nil
	171	case l:
	172	return nil
	173	default:
	174	return errors.New("mismatch in number of column values")
	175	}
	176	}
	177
	178	for _, pv := range pvs {
	179	switch {
	180	case pv.Key != "" \|\| !pv.Value.IsNull():
	181	continue
	182	case pv.Values != nil:
	183	if err := setCount(len(pv.Values)); err != nil {
	184	return 0, err
	185	}
	186	case pv.ListKey != "":
	187	bv, err := pv.lookupList(bindVars)
	188	if err != nil {
	189	return 0, err
	190	}
	191	if err := setCount(len(bv.Values)); err != nil {
	192	return 0, err
	193	}
	194	}
	195	}
	196
	197	if count == -1 {
	198	// If there were no lists inside, it was a single row.
	199	// Note that count can never be 0 because there is enough
	200	// protection at the top level: list bind vars must have
	201	// at least one value (enforced by vtgate), and AST lists
	202	// must have at least one value (enforced by the parser).
	203	// Also lists created internally after vtgate validation
	204	// ensure at least one value.
	205	// TODO(sougou): verify and change API to enforce this.
	206	return 1, nil
	207	}
	208	return count, nil
	209	}
	210
	211	// ResolveRows resolves a []PlanValue as rows based on the supplied bindvars.
	212	func ResolveRows(pvs []PlanValue, bindVars map[string]*querypb.BindVariable) ([][]Value, error) {
	213	count, err := rowCount(pvs, bindVars)
	214	if err != nil {
	215	return nil, err
	216	}
	217
	218	// Allocate the rows.
	219	rows := make([][]Value, count)
	220	for i := range rows {
	221	rows[i] = make([]Value, len(pvs))
	222	}
	223
	224	// Using j becasue we're resolving by columns.
	225	for j, pv := range pvs {
	226	switch {
	227	case pv.Key != "":
	228	bv, err := pv.lookupValue(bindVars)
	229	if err != nil {
	230	return nil, err
	231	}
	232	for i := range rows {
	233	rows[i][j] = MakeTrusted(bv.Type, bv.Value)
	234	}
	235	case !pv.Value.IsNull():
	236	for i := range rows {
	237	rows[i][j] = pv.Value
	238	}
	239	case pv.ListKey != "":
	240	bv, err := pv.lookupList(bindVars)
	241	if err != nil {
	242	// This code is unreachable because pvRowCount already checks this.
	243	return nil, err
	244	}
	245	for i := range rows {
	246	rows[i][j] = MakeTrusted(bv.Values[i].Type, bv.Values[i].Value)
	247	}
	248	case pv.Values != nil:
	249	for i := range rows {
	250	rows[i][j], err = pv.Values[i].ResolveValue(bindVars)
	251	if err != nil {
	252	return nil, err
	253	}
	254	}
	255	// default case is a NULL value, which the row values are already initialized to.
	256	}
	257	}
	258	return rows, nil
	259	}

vendor/github.com/xwb1989/sqlparser/dependency/sqltypes/testing.go+154−0

@@ -0,0 +1,154 @@
	1	/*
	2	Copyright 2017 Google Inc.
	3
	4	Licensed under the Apache License, Version 2.0 (the "License");
	5	you may not use this file except in compliance with the License.
	6	You may obtain a copy of the License at
	7
	8	http://www.apache.org/licenses/LICENSE-2.0
	9
	10	Unless required by applicable law or agreed to in writing, software
	11	distributed under the License is distributed on an "AS IS" BASIS,
	12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	See the License for the specific language governing permissions and
	14	limitations under the License.
	15	*/
	16
	17	package sqltypes
	18
	19	import (
	20	querypb "github.com/xwb1989/sqlparser/dependency/querypb"
	21	)
	22
	23	// Functions in this file should only be used for testing.
	24	// This is an experiment to see if test code bloat can be
	25	// reduced and readability improved.
	26
	27	/*
	28	// MakeTestFields builds a []*querypb.Field for testing.
	29	// fields := sqltypes.MakeTestFields(
	30	// "a\|b",
	31	// "int64\|varchar",
	32	// )
	33	// The field types are as defined in querypb and are case
	34	// insensitive. Column delimiters must be used only to sepearate
	35	// strings and not at the beginning or the end.
	36	func MakeTestFields(names, types string) []*querypb.Field {
	37	n := split(names)
	38	t := split(types)
	39	var fields []*querypb.Field
	40	for i := range n {
	41	fields = append(fields, &querypb.Field{
	42	Name: n[i],
	43	Type: querypb.Type(querypb.Type_value[strings.ToUpper(t[i])]),
	44	})
	45	}
	46	return fields
	47	}
	48
	49	// MakeTestResult builds a *sqltypes.Result object for testing.
	50	// result := sqltypes.MakeTestResult(
	51	// fields,
	52	// " 1\|a",
	53	// "10\|abcd",
	54	// )
	55	// The field type values are set as the types for the rows built.
	56	// Spaces are trimmed from row values. "null" is treated as NULL.
	57	func MakeTestResult(fields []querypb.Field, rows ...string) Result {
	58	result := &Result{
	59	Fields: fields,
	60	}
	61	if len(rows) > 0 {
	62	result.Rows = make([][]Value, len(rows))
	63	}
	64	for i, row := range rows {
	65	result.Rows[i] = make([]Value, len(fields))
	66	for j, col := range split(row) {
	67	if col == "null" {
	68	continue
	69	}
	70	result.Rows[i][j] = MakeTrusted(fields[j].Type, []byte(col))
	71	}
	72	}
	73	result.RowsAffected = uint64(len(result.Rows))
	74	return result
	75	}
	76
	77	// MakeTestStreamingResults builds a list of results for streaming.
	78	// results := sqltypes.MakeStreamingResults(
	79	// fields,
	80	// "1\|a",
	81	// "2\|b",
	82	// "---",
	83	// "c\|c",
	84	// )
	85	// The first result contains only the fields. Subsequent results
	86	// are built using the field types. Every input that starts with a "-"
	87	// is treated as streaming delimiter for one result. A final
	88	// delimiter must not be supplied.
	89	func MakeTestStreamingResults(fields []querypb.Field, rows ...string) []Result {
	90	var results []*Result
	91	results = append(results, &Result{Fields: fields})
	92	start := 0
	93	cur := 0
	94	// Add a final streaming delimiter to simplify the loop below.
	95	rows = append(rows, "-")
	96	for cur < len(rows) {
	97	if rows[cur][0] != '-' {
	98	cur++
	99	continue
	100	}
	101	result := MakeTestResult(fields, rows[start:cur]...)
	102	result.Fields = nil
	103	result.RowsAffected = 0
	104	results = append(results, result)
	105	start = cur + 1
	106	cur = start
	107	}
	108	return results
	109	}
	110	*/
	111
	112	// TestBindVariable makes a *querypb.BindVariable from
	113	// an interface{}.It panics on invalid input.
	114	// This function should only be used for testing.
	115	func TestBindVariable(v interface{}) *querypb.BindVariable {
	116	if v == nil {
	117	return NullBindVariable
	118	}
	119	bv, err := BuildBindVariable(v)
	120	if err != nil {
	121	panic(err)
	122	}
	123	return bv
	124	}
	125
	126	// TestValue builds a Value from typ and val.
	127	// This function should only be used for testing.
	128	func TestValue(typ querypb.Type, val string) Value {
	129	return MakeTrusted(typ, []byte(val))
	130	}
	131
	132	/*
	133	// PrintResults prints []*Results into a string.
	134	// This function should only be used for testing.
	135	func PrintResults(results []*Result) string {
	136	b := new(bytes.Buffer)
	137	for i, r := range results {
	138	if i == 0 {
	139	fmt.Fprintf(b, "%v", r)
	140	continue
	141	}
	142	fmt.Fprintf(b, ", %v", r)
	143	}
	144	return b.String()
	145	}
	146
	147	func split(str string) []string {
	148	splits := strings.Split(str, "\|")
	149	for i, v := range splits {
	150	splits[i] = strings.TrimSpace(v)
	151	}
	152	return splits
	153	}
	154	*/

vendor/github.com/xwb1989/sqlparser/dependency/sqltypes/type.go+288−0

@@ -0,0 +1,288 @@
	1	/*
	2	Copyright 2017 Google Inc.
	3
	4	Licensed under the Apache License, Version 2.0 (the "License");
	5	you may not use this file except in compliance with the License.
	6	You may obtain a copy of the License at
	7
	8	http://www.apache.org/licenses/LICENSE-2.0
	9
	10	Unless required by applicable law or agreed to in writing, software
	11	distributed under the License is distributed on an "AS IS" BASIS,
	12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	See the License for the specific language governing permissions and
	14	limitations under the License.
	15	*/
	16
	17	package sqltypes
	18
	19	import (
	20	"fmt"
	21
	22	"github.com/xwb1989/sqlparser/dependency/querypb"
	23	)
	24
	25	// This file provides wrappers and support
	26	// functions for querypb.Type.
	27
	28	// These bit flags can be used to query on the
	29	// common properties of types.
	30	const (
	31	flagIsIntegral = int(querypb.Flag_ISINTEGRAL)
	32	flagIsUnsigned = int(querypb.Flag_ISUNSIGNED)
	33	flagIsFloat = int(querypb.Flag_ISFLOAT)
	34	flagIsQuoted = int(querypb.Flag_ISQUOTED)
	35	flagIsText = int(querypb.Flag_ISTEXT)
	36	flagIsBinary = int(querypb.Flag_ISBINARY)
	37	)
	38
	39	// IsIntegral returns true if querypb.Type is an integral
	40	// (signed/unsigned) that can be represented using
	41	// up to 64 binary bits.
	42	// If you have a Value object, use its member function.
	43	func IsIntegral(t querypb.Type) bool {
	44	return int(t)&flagIsIntegral == flagIsIntegral
	45	}
	46
	47	// IsSigned returns true if querypb.Type is a signed integral.
	48	// If you have a Value object, use its member function.
	49	func IsSigned(t querypb.Type) bool {
	50	return int(t)&(flagIsIntegral\|flagIsUnsigned) == flagIsIntegral
	51	}
	52
	53	// IsUnsigned returns true if querypb.Type is an unsigned integral.
	54	// Caution: this is not the same as !IsSigned.
	55	// If you have a Value object, use its member function.
	56	func IsUnsigned(t querypb.Type) bool {
	57	return int(t)&(flagIsIntegral\|flagIsUnsigned) == flagIsIntegral\|flagIsUnsigned
	58	}
	59
	60	// IsFloat returns true is querypb.Type is a floating point.
	61	// If you have a Value object, use its member function.
	62	func IsFloat(t querypb.Type) bool {
	63	return int(t)&flagIsFloat == flagIsFloat
	64	}
	65
	66	// IsQuoted returns true if querypb.Type is a quoted text or binary.
	67	// If you have a Value object, use its member function.
	68	func IsQuoted(t querypb.Type) bool {
	69	return int(t)&flagIsQuoted == flagIsQuoted
	70	}
	71
	72	// IsText returns true if querypb.Type is a text.
	73	// If you have a Value object, use its member function.
	74	func IsText(t querypb.Type) bool {
	75	return int(t)&flagIsText == flagIsText
	76	}
	77
	78	// IsBinary returns true if querypb.Type is a binary.
	79	// If you have a Value object, use its member function.
	80	func IsBinary(t querypb.Type) bool {
	81	return int(t)&flagIsBinary == flagIsBinary
	82	}
	83
	84	// isNumber returns true if the type is any type of number.
	85	func isNumber(t querypb.Type) bool {
	86	return IsIntegral(t) \|\| IsFloat(t) \|\| t == Decimal
	87	}
	88
	89	// Vitess data types. These are idiomatically
	90	// named synonyms for the querypb.Type values.
	91	// Although these constants are interchangeable,
	92	// they should be treated as different from querypb.Type.
	93	// Use the synonyms only to refer to the type in Value.
	94	// For proto variables, use the querypb.Type constants
	95	// instead.
	96	// The following conditions are non-overlapping
	97	// and cover all types: IsSigned(), IsUnsigned(),
	98	// IsFloat(), IsQuoted(), Null, Decimal, Expression.
	99	// Also, IsIntegral() == (IsSigned()\|\|IsUnsigned()).
	100	// TestCategory needs to be updated accordingly if
	101	// you add a new type.
	102	// If IsBinary or IsText is true, then IsQuoted is
	103	// also true. But there are IsQuoted types that are
	104	// neither binary or text.
	105	// querypb.Type_TUPLE is not included in this list
	106	// because it's not a valid Value type.
	107	// TODO(sougou): provide a categorization function
	108	// that returns enums, which will allow for cleaner
	109	// switch statements for those who want to cover types
	110	// by their category.
	111	const (
	112	Null = querypb.Type_NULL_TYPE
	113	Int8 = querypb.Type_INT8
	114	Uint8 = querypb.Type_UINT8
	115	Int16 = querypb.Type_INT16
	116	Uint16 = querypb.Type_UINT16
	117	Int24 = querypb.Type_INT24
	118	Uint24 = querypb.Type_UINT24
	119	Int32 = querypb.Type_INT32
	120	Uint32 = querypb.Type_UINT32
	121	Int64 = querypb.Type_INT64
	122	Uint64 = querypb.Type_UINT64
	123	Float32 = querypb.Type_FLOAT32
	124	Float64 = querypb.Type_FLOAT64
	125	Timestamp = querypb.Type_TIMESTAMP
	126	Date = querypb.Type_DATE
	127	Time = querypb.Type_TIME
	128	Datetime = querypb.Type_DATETIME
	129	Year = querypb.Type_YEAR
	130	Decimal = querypb.Type_DECIMAL
	131	Text = querypb.Type_TEXT
	132	Blob = querypb.Type_BLOB
	133	VarChar = querypb.Type_VARCHAR
	134	VarBinary = querypb.Type_VARBINARY
	135	Char = querypb.Type_CHAR
	136	Binary = querypb.Type_BINARY
	137	Bit = querypb.Type_BIT
	138	Enum = querypb.Type_ENUM
	139	Set = querypb.Type_SET
	140	Geometry = querypb.Type_GEOMETRY
	141	TypeJSON = querypb.Type_JSON
	142	Expression = querypb.Type_EXPRESSION
	143	)
	144
	145	// bit-shift the mysql flags by two byte so we
	146	// can merge them with the mysql or vitess types.
	147	const (
	148	mysqlUnsigned = 32
	149	mysqlBinary = 128
	150	mysqlEnum = 256
	151	mysqlSet = 2048
	152	)
	153
	154	// If you add to this map, make sure you add a test case
	155	// in tabletserver/endtoend.
	156	var mysqlToType = map[int64]querypb.Type{
	157	1: Int8,
	158	2: Int16,
	159	3: Int32,
	160	4: Float32,
	161	5: Float64,
	162	6: Null,
	163	7: Timestamp,
	164	8: Int64,
	165	9: Int24,
	166	10: Date,
	167	11: Time,
	168	12: Datetime,
	169	13: Year,
	170	16: Bit,
	171	245: TypeJSON,
	172	246: Decimal,
	173	249: Text,
	174	250: Text,
	175	251: Text,
	176	252: Text,
	177	253: VarChar,
	178	254: Char,
	179	255: Geometry,
	180	}
	181
	182	// modifyType modifies the vitess type based on the
	183	// mysql flag. The function checks specific flags based
	184	// on the type. This allows us to ignore stray flags
	185	// that MySQL occasionally sets.
	186	func modifyType(typ querypb.Type, flags int64) querypb.Type {
	187	switch typ {
	188	case Int8:
	189	if flags&mysqlUnsigned != 0 {
	190	return Uint8
	191	}
	192	return Int8
	193	case Int16:
	194	if flags&mysqlUnsigned != 0 {
	195	return Uint16
	196	}
	197	return Int16
	198	case Int32:
	199	if flags&mysqlUnsigned != 0 {
	200	return Uint32
	201	}
	202	return Int32
	203	case Int64:
	204	if flags&mysqlUnsigned != 0 {
	205	return Uint64
	206	}
	207	return Int64
	208	case Int24:
	209	if flags&mysqlUnsigned != 0 {
	210	return Uint24
	211	}
	212	return Int24
	213	case Text:
	214	if flags&mysqlBinary != 0 {
	215	return Blob
	216	}
	217	return Text
	218	case VarChar:
	219	if flags&mysqlBinary != 0 {
	220	return VarBinary
	221	}
	222	return VarChar
	223	case Char:
	224	if flags&mysqlBinary != 0 {
	225	return Binary
	226	}
	227	if flags&mysqlEnum != 0 {
	228	return Enum
	229	}
	230	if flags&mysqlSet != 0 {
	231	return Set
	232	}
	233	return Char
	234	}
	235	return typ
	236	}
	237
	238	// MySQLToType computes the vitess type from mysql type and flags.
	239	func MySQLToType(mysqlType, flags int64) (typ querypb.Type, err error) {
	240	result, ok := mysqlToType[mysqlType]
	241	if !ok {
	242	return 0, fmt.Errorf("unsupported type: %d", mysqlType)
	243	}
	244	return modifyType(result, flags), nil
	245	}
	246
	247	// typeToMySQL is the reverse of mysqlToType.
	248	var typeToMySQL = map[querypb.Type]struct {
	249	typ int64
	250	flags int64
	251	}{
	252	Int8: {typ: 1},
	253	Uint8: {typ: 1, flags: mysqlUnsigned},
	254	Int16: {typ: 2},
	255	Uint16: {typ: 2, flags: mysqlUnsigned},
	256	Int32: {typ: 3},
	257	Uint32: {typ: 3, flags: mysqlUnsigned},
	258	Float32: {typ: 4},
	259	Float64: {typ: 5},
	260	Null: {typ: 6, flags: mysqlBinary},
	261	Timestamp: {typ: 7},
	262	Int64: {typ: 8},
	263	Uint64: {typ: 8, flags: mysqlUnsigned},
	264	Int24: {typ: 9},
	265	Uint24: {typ: 9, flags: mysqlUnsigned},
	266	Date: {typ: 10, flags: mysqlBinary},
	267	Time: {typ: 11, flags: mysqlBinary},
	268	Datetime: {typ: 12, flags: mysqlBinary},
	269	Year: {typ: 13, flags: mysqlUnsigned},
	270	Bit: {typ: 16, flags: mysqlUnsigned},
	271	TypeJSON: {typ: 245},
	272	Decimal: {typ: 246},
	273	Text: {typ: 252},
	274	Blob: {typ: 252, flags: mysqlBinary},
	275	VarChar: {typ: 253},
	276	VarBinary: {typ: 253, flags: mysqlBinary},
	277	Char: {typ: 254},
	278	Binary: {typ: 254, flags: mysqlBinary},
	279	Enum: {typ: 254, flags: mysqlEnum},
	280	Set: {typ: 254, flags: mysqlSet},
	281	Geometry: {typ: 255},
	282	}
	283
	284	// TypeToMySQL returns the equivalent mysql type and flag for a vitess type.
	285	func TypeToMySQL(typ querypb.Type) (mysqlType, flags int64) {
	286	val := typeToMySQL[typ]
	287	return val.typ, val.flags
	288	}

vendor/github.com/xwb1989/sqlparser/dependency/sqltypes/value.go+376−0

@@ -0,0 +1,376 @@
	1	/*
	2	Copyright 2017 Google Inc.
	3
	4	Licensed under the Apache License, Version 2.0 (the "License");
	5	you may not use this file except in compliance with the License.
	6	You may obtain a copy of the License at
	7
	8	http://www.apache.org/licenses/LICENSE-2.0
	9
	10	Unless required by applicable law or agreed to in writing, software
	11	distributed under the License is distributed on an "AS IS" BASIS,
	12	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	See the License for the specific language governing permissions and
	14	limitations under the License.
	15	*/
	16
	17	// Package sqltypes implements interfaces and types that represent SQL values.
	18	package sqltypes
	19
	20	import (
	21	"encoding/base64"
	22	"encoding/json"
	23	"fmt"
	24	"strconv"
	25
	26	"github.com/xwb1989/sqlparser/dependency/bytes2"
	27	"github.com/xwb1989/sqlparser/dependency/hack"
	28
	29	"github.com/xwb1989/sqlparser/dependency/querypb"
	30	)
	31
	32	var (
	33	// NULL represents the NULL value.
	34	NULL = Value{}
	35
	36	// DontEscape tells you if a character should not be escaped.
	37	DontEscape = byte(255)
	38
	39	nullstr = []byte("null")
	40	)
	41
	42	// BinWriter interface is used for encoding values.
	43	// Types like bytes.Buffer conform to this interface.
	44	// We expect the writer objects to be in-memory buffers.
	45	// So, we don't expect the write operations to fail.
	46	type BinWriter interface {
	47	Write([]byte) (int, error)
	48	}
	49
	50	// Value can store any SQL value. If the value represents
	51	// an integral type, the bytes are always stored as a cannonical
	52	// representation that matches how MySQL returns such values.
	53	type Value struct {
	54	typ querypb.Type
	55	val []byte
	56	}
	57
	58	// NewValue builds a Value using typ and val. If the value and typ
	59	// don't match, it returns an error.
	60	func NewValue(typ querypb.Type, val []byte) (v Value, err error) {
	61	switch {
	62	case IsSigned(typ):
	63	if _, err := strconv.ParseInt(string(val), 0, 64); err != nil {
	64	return NULL, err
	65	}
	66	return MakeTrusted(typ, val), nil
	67	case IsUnsigned(typ):
	68	if _, err := strconv.ParseUint(string(val), 0, 64); err != nil {
	69	return NULL, err
	70	}
	71	return MakeTrusted(typ, val), nil
	72	case IsFloat(typ) \|\| typ == Decimal:
	73	if _, err := strconv.ParseFloat(string(val), 64); err != nil {
	74	return NULL, err
	75	}
	76	return MakeTrusted(typ, val), nil
	77	case IsQuoted(typ) \|\| typ == Null:
	78	return MakeTrusted(typ, val), nil
	79	}
	80	// All other types are unsafe or invalid.
	81	return NULL, fmt.Errorf("invalid type specified for MakeValue: %v", typ)
	82	}
	83
	84	// MakeTrusted makes a new Value based on the type.
	85	// This function should only be used if you know the value
	86	// and type conform to the rules. Every place this function is
	87	// called, a comment is needed that explains why it's justified.
	88	// Exceptions: The current package and mysql package do not need
	89	// comments. Other packages can also use the function to create
	90	// VarBinary or VarChar values.
	91	func MakeTrusted(typ querypb.Type, val []byte) Value {
	92	if typ == Null {
	93	return NULL
	94	}
	95	return Value{typ: typ, val: val}
	96	}
	97
	98	// NewInt64 builds an Int64 Value.
	99	func NewInt64(v int64) Value {
	100	return MakeTrusted(Int64, strconv.AppendInt(nil, v, 10))
	101	}
	102
	103	// NewInt32 builds an Int64 Value.
	104	func NewInt32(v int32) Value {
	105	return MakeTrusted(Int32, strconv.AppendInt(nil, int64(v), 10))
	106	}
	107
	108	// NewUint64 builds an Uint64 Value.
	109	func NewUint64(v uint64) Value {
	110	return MakeTrusted(Uint64, strconv.AppendUint(nil, v, 10))
	111	}
	112
	113	// NewFloat64 builds an Float64 Value.
	114	func NewFloat64(v float64) Value {
	115	return MakeTrusted(Float64, strconv.AppendFloat(nil, v, 'g', -1, 64))
	116	}
	117
	118	// NewVarChar builds a VarChar Value.
	119	func NewVarChar(v string) Value {
	120	return MakeTrusted(VarChar, []byte(v))
	121	}
	122
	123	// NewVarBinary builds a VarBinary Value.
	124	// The input is a string because it's the most common use case.
	125	func NewVarBinary(v string) Value {
	126	return MakeTrusted(VarBinary, []byte(v))
	127	}
	128
	129	// NewIntegral builds an integral type from a string representaion.
	130	// The type will be Int64 or Uint64. Int64 will be preferred where possible.
	131	func NewIntegral(val string) (n Value, err error) {
	132	signed, err := strconv.ParseInt(val, 0, 64)
	133	if err == nil {
	134	return MakeTrusted(Int64, strconv.AppendInt(nil, signed, 10)), nil
	135	}
	136	unsigned, err := strconv.ParseUint(val, 0, 64)
	137	if err != nil {
	138	return Value{}, err
	139	}
	140	return MakeTrusted(Uint64, strconv.AppendUint(nil, unsigned, 10)), nil
	141	}
	142
	143	// InterfaceToValue builds a value from a go type.
	144	// Supported types are nil, int64, uint64, float64,
	145	// string and []byte.
	146	// This function is deprecated. Use the type-specific
	147	// functions instead.
	148	func InterfaceToValue(goval interface{}) (Value, error) {
	149	switch goval := goval.(type) {
	150	case nil:
	151	return NULL, nil
	152	case []byte:
	153	return MakeTrusted(VarBinary, goval), nil
	154	case int64:
	155	return NewInt64(goval), nil
	156	case uint64:
	157	return NewUint64(goval), nil
	158	case float64:
	159	return NewFloat64(goval), nil
	160	case string:
	161	return NewVarChar(goval), nil
	162	default:
	163	return NULL, fmt.Errorf("unexpected type %T: %v", goval, goval)
	164	}
	165	}
	166
	167	// Type returns the type of Value.
	168	func (v Value) Type() querypb.Type {
	169	return v.typ
	170	}
	171
	172	// Raw returns the internal represenation of the value. For newer types,
	173	// this may not match MySQL's representation.
	174	func (v Value) Raw() []byte {
	175	return v.val
	176	}
	177
	178	// ToBytes returns the value as MySQL would return it as []byte.
	179	// In contrast, Raw returns the internal representation of the Value, which may not
	180	// match MySQL's representation for newer types.
	181	// If the value is not convertible like in the case of Expression, it returns nil.
	182	func (v Value) ToBytes() []byte {
	183	if v.typ == Expression {
	184	return nil
	185	}
	186	return v.val
	187	}
	188
	189	// Len returns the length.
	190	func (v Value) Len() int {
	191	return len(v.val)
	192	}
	193
	194	// ToString returns the value as MySQL would return it as string.
	195	// If the value is not convertible like in the case of Expression, it returns nil.
	196	func (v Value) ToString() string {
	197	if v.typ == Expression {
	198	return ""
	199	}
	200	return hack.String(v.val)
	201	}
	202
	203	// String returns a printable version of the value.
	204	func (v Value) String() string {
	205	if v.typ == Null {
	206	return "NULL"
	207	}
	208	if v.IsQuoted() {
	209	return fmt.Sprintf("%v(%q)", v.typ, v.val)
	210	}
	211	return fmt.Sprintf("%v(%s)", v.typ, v.val)
	212	}
	213
	214	// EncodeSQL encodes the value into an SQL statement. Can be binary.
	215	func (v Value) EncodeSQL(b BinWriter) {
	216	switch {
	217	case v.typ == Null:
	218	b.Write(nullstr)
	219	case v.IsQuoted():
	220	encodeBytesSQL(v.val, b)
	221	default:
	222	b.Write(v.val)
	223	}
	224	}
	225
	226	// EncodeASCII encodes the value using 7-bit clean ascii bytes.
	227	func (v Value) EncodeASCII(b BinWriter) {
	228	switch {
	229	case v.typ == Null:
	230	b.Write(nullstr)
	231	case v.IsQuoted():
	232	encodeBytesASCII(v.val, b)
	233	default:
	234	b.Write(v.val)
	235	}
	236	}
	237
	238	// IsNull returns true if Value is null.
	239	func (v Value) IsNull() bool {
	240	return v.typ == Null
	241	}
	242
	243	// IsIntegral returns true if Value is an integral.
	244	func (v Value) IsIntegral() bool {
	245	return IsIntegral(v.typ)
	246	}
	247
	248	// IsSigned returns true if Value is a signed integral.
	249	func (v Value) IsSigned() bool {
	250	return IsSigned(v.typ)
	251	}
	252
	253	// IsUnsigned returns true if Value is an unsigned integral.
	254	func (v Value) IsUnsigned() bool {
	255	return IsUnsigned(v.typ)
	256	}
	257
	258	// IsFloat returns true if Value is a float.
	259	func (v Value) IsFloat() bool {
	260	return IsFloat(v.typ)
	261	}
	262
	263	// IsQuoted returns true if Value must be SQL-quoted.
	264	func (v Value) IsQuoted() bool {
	265	return IsQuoted(v.typ)
	266	}
	267
	268	// IsText returns true if Value is a collatable text.
	269	func (v Value) IsText() bool {
	270	return IsText(v.typ)
	271	}
	272
	273	// IsBinary returns true if Value is binary.
	274	func (v Value) IsBinary() bool {
	275	return IsBinary(v.typ)
	276	}
	277
	278	// MarshalJSON should only be used for testing.
	279	// It's not a complete implementation.
	280	func (v Value) MarshalJSON() ([]byte, error) {
	281	switch {
	282	case v.IsQuoted():
	283	return json.Marshal(v.ToString())
	284	case v.typ == Null:
	285	return nullstr, nil
	286	}
	287	return v.val, nil
	288	}
	289
	290	// UnmarshalJSON should only be used for testing.
	291	// It's not a complete implementation.
	292	func (v *Value) UnmarshalJSON(b []byte) error {
	293	if len(b) == 0 {
	294	return fmt.Errorf("error unmarshaling empty bytes")
	295	}
	296	var val interface{}
	297	var err error
	298	switch b[0] {
	299	case '-':
	300	var ival int64
	301	err = json.Unmarshal(b, &ival)
	302	val = ival
	303	case '"':
	304	var bval []byte
	305	err = json.Unmarshal(b, &bval)
	306	val = bval
	307	case 'n': // null
	308	err = json.Unmarshal(b, &val)
	309	default:
	310	var uval uint64
	311	err = json.Unmarshal(b, &uval)
	312	val = uval