History

S3 Select API Support for CSV (#6127)

Add support for trivial where clause cases

Arjun Mishra committed 8 years ago

Commit 7c14cdb1 parent 0e0232859 files · +30860 −3Browse files

Unified · Split

59 files
  1. Makefile +2 −2
  2. cmd/api-datatypes.go +81 −0
  3. cmd/api-errors.go +679 −0
  4. cmd/api-router.go +2 −0
  5. cmd/generic-handlers.go +0 −1
  6. cmd/object-handlers.go +187 −0
  7. docs/select/README.md +98 −0
  8. pkg/s3select/errors.go +486 −0
  9. pkg/s3select/funcEval.go +231 −0
  10. pkg/s3select/helpers.go +754 −0
  11. pkg/s3select/input.go +381 −0
  12. pkg/s3select/output.go +460 −0
  13. pkg/s3select/select.go +415 −0
  14. pkg/s3select/select_test.go +1070 −0
  15. vendor/github.com/klauspost/compress/LICENSE +27 −0
  16. vendor/github.com/klauspost/compress/flate/copy.go +32 −0
  17. vendor/github.com/klauspost/compress/flate/crc32_amd64.go +41 −0
  18. vendor/github.com/klauspost/compress/flate/crc32_amd64.s +213 −0
  19. vendor/github.com/klauspost/compress/flate/crc32_noasm.go +35 −0
  20. vendor/github.com/klauspost/compress/flate/deflate.go +1353 −0
  21. vendor/github.com/klauspost/compress/flate/dict_decoder.go +184 −0
  22. vendor/github.com/klauspost/compress/flate/gen.go +265 −0
  23. vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go +701 −0
  24. vendor/github.com/klauspost/compress/flate/huffman_code.go +344 −0
  25. vendor/github.com/klauspost/compress/flate/inflate.go +868 −0
  26. vendor/github.com/klauspost/compress/flate/reverse_bits.go +48 −0
  27. vendor/github.com/klauspost/compress/flate/snappy.go +900 −0
  28. vendor/github.com/klauspost/compress/flate/token.go +115 −0
  29. vendor/github.com/klauspost/pgzip/GO_LICENSE +27 −0
  30. vendor/github.com/klauspost/pgzip/LICENSE +22 −0
  31. vendor/github.com/klauspost/pgzip/README.md +136 −0
  32. vendor/github.com/klauspost/pgzip/circle.yml +7 −0
  33. vendor/github.com/klauspost/pgzip/gunzip.go +573 −0
  34. vendor/github.com/klauspost/pgzip/gzip.go +501 −0
  35. vendor/github.com/xwb1989/sqlparser/CONTRIBUTORS.md +9 −0
  36. vendor/github.com/xwb1989/sqlparser/LICENSE.md +201 −0
  37. vendor/github.com/xwb1989/sqlparser/Makefile +22 −0
  38. vendor/github.com/xwb1989/sqlparser/README.md +150 −0
  39. vendor/github.com/xwb1989/sqlparser/analyzer.go +343 −0
  40. vendor/github.com/xwb1989/sqlparser/ast.go +3450 −0
  41. vendor/github.com/xwb1989/sqlparser/comments.go +293 −0
  42. vendor/github.com/xwb1989/sqlparser/dependency/bytes2/buffer.go +65 −0
  43. vendor/github.com/xwb1989/sqlparser/dependency/hack/hack.go +79 −0
  44. vendor/github.com/xwb1989/sqlparser/dependency/querypb/query.pb.go +2734 −0
  45. vendor/github.com/xwb1989/sqlparser/dependency/sqltypes/bind_variables.go +266 −0
  46. vendor/github.com/xwb1989/sqlparser/dependency/sqltypes/plan_value.go +259 −0
  47. vendor/github.com/xwb1989/sqlparser/dependency/sqltypes/testing.go +154 −0
  48. vendor/github.com/xwb1989/sqlparser/dependency/sqltypes/type.go +288 −0
  49. vendor/github.com/xwb1989/sqlparser/dependency/sqltypes/value.go +376 −0
  50. vendor/github.com/xwb1989/sqlparser/encodable.go +99 −0
  51. vendor/github.com/xwb1989/sqlparser/impossible_query.go +39 −0
  52. vendor/github.com/xwb1989/sqlparser/normalizer.go +224 −0
  53. vendor/github.com/xwb1989/sqlparser/parsed_query.go +119 −0
  54. vendor/github.com/xwb1989/sqlparser/redact_query.go +19 −0
  55. vendor/github.com/xwb1989/sqlparser/sql.go +6136 −0
  56. vendor/github.com/xwb1989/sqlparser/sql.y +3159 −0
  57. vendor/github.com/xwb1989/sqlparser/token.go +950 −0
  58. vendor/github.com/xwb1989/sqlparser/tracked_buffer.go +140 −0
  59. vendor/vendor.json +48 −0
Makefile+2−2
@@ -42,8 +42,8 @@ ineffassign: · 41 unmodified lines
4242
4343 cyclo:
4444 @echo "Running $@"
45− @${GOPATH}/bin/gocyclo -over 100 cmd
46− @${GOPATH}/bin/gocyclo -over 100 pkg
45+ @${GOPATH}/bin/gocyclo -over 200 cmd
46+ @${GOPATH}/bin/gocyclo -over 200 pkg
4747
4848 deadcode:
4949 @echo "Running $@"
cmd/api-datatypes.go+81−0
@@ -25,6 +25,87 @@ const ( · 24 unmodified lines
2525 responseRequestIDKey = "x-amz-request-id"
2626 )
2727
28+// CSVFileHeaderInfo -Can be either USE IGNORE OR NONE, defines what to do with
29+// the first row
30+type CSVFileHeaderInfo string
31+
32+// Constants for file header info.
33+const (
34+ CSVFileHeaderInfoNone CSVFileHeaderInfo = "NONE"
35+ CSVFileHeaderInfoIgnore = "IGNORE"
36+ CSVFileHeaderInfoUse = "USE"
37+)
38+
39+// SelectCompressionType - ONLY GZIP is supported
40+type SelectCompressionType string
41+
42+// Constants for compression types under select API.
43+const (
44+ SelectCompressionNONE SelectCompressionType = "NONE"
45+ SelectCompressionGZIP = "GZIP"
46+ SelectCompressionBZIP = "BZIP2"
47+)
48+
49+// CSVQuoteFields - Can be either Always or AsNeeded
50+type CSVQuoteFields string
51+
52+// Constants for csv quote styles.
53+const (
54+ CSVQuoteFieldsAlways CSVQuoteFields = "Always"
55+ CSVQuoteFieldsAsNeeded = "AsNeeded"
56+)
57+
58+// QueryExpressionType - Currently can only be SQL
59+type QueryExpressionType string
60+
61+// Constants for expression type.
62+const (
63+ QueryExpressionTypeSQL QueryExpressionType = "SQL"
64+)
65+
66+// JSONType determines json input serialization type.
67+type JSONType string
68+
69+// Constants for JSONTypes.
70+const (
71+ JSONDocumentType JSONType = "Document"
72+ JSONStreamType = "Stream"
73+ JSONLinesType = "Lines"
74+)
75+
76+// ObjectSelectRequest - represents the input select body
77+type ObjectSelectRequest struct {
78+ XMLName xml.Name `xml:"SelectObjectContentRequest" json:"-"`
79+ Expression string
80+ ExpressionType QueryExpressionType
81+ InputSerialization struct {
82+ CompressionType SelectCompressionType
83+ CSV *struct {
84+ FileHeaderInfo CSVFileHeaderInfo
85+ RecordDelimiter string
86+ FieldDelimiter string
87+ QuoteCharacter string
88+ QuoteEscapeCharacter string
89+ Comments string
90+ }
91+ JSON *struct {
92+ Type JSONType
93+ }
94+ }
95+ OutputSerialization struct {
96+ CSV *struct {
97+ QuoteFields CSVQuoteFields
98+ RecordDelimiter string
99+ FieldDelimiter string
100+ QuoteCharacter string
101+ QuoteEscapeCharacter string
102+ }
103+ JSON *struct {
104+ RecordDelimiter string
105+ }
106+ }
107+}
108+
28109 // ObjectIdentifier carries key name for the object to delete.
29110 type ObjectIdentifier struct {
30111 ObjectName string `xml:"Key"`
cmd/api-errors.go+679−0
@@ -27,6 +27,7 @@ import ( · 26 unmodified lines
2727 "github.com/minio/minio/pkg/dns"
2828 "github.com/minio/minio/pkg/event"
2929 "github.com/minio/minio/pkg/hash"
30+ "github.com/minio/minio/pkg/s3select"
3031 )
3132
3233 // APIError structure
@@ -190,6 +191,7 @@ const ( · 157 unmodified lines
190191 ErrAdminCredentialsMismatch
191192 ErrInsecureClientRequest
192193 ErrObjectTampered
194+
193195 ErrHealNotImplemented
194196 ErrHealNoSuchProcess
195197 ErrHealInvalidClientToken
@@ -197,6 +199,93 @@ const ( · 1 unmodified line
197199 ErrHealAlreadyRunning
198200 ErrHealOverlappingPaths
199201 ErrIncorrectContinuationToken
202+
203+ //S3 Select Errors
204+ ErrEmptyRequestBody
205+ ErrUnsupportedFunction
206+ ErrInvalidExpressionType
207+ ErrBusy
208+ ErrUnauthorizedAccess
209+ ErrExpressionTooLong
210+ ErrIllegalSQLFunctionArgument
211+ ErrInvalidKeyPath
212+ ErrInvalidCompressionFormat
213+ ErrInvalidFileHeaderInfo
214+ ErrInvalidJSONType
215+ ErrInvalidQuoteFields
216+ ErrInvalidRequestParameter
217+ ErrInvalidDataType
218+ ErrInvalidTextEncoding
219+ ErrInvalidDataSource
220+ ErrInvalidTableAlias
221+ ErrMissingRequiredParameter
222+ ErrObjectSerializationConflict
223+ ErrUnsupportedSQLOperation
224+ ErrUnsupportedSQLStructure
225+ ErrUnsupportedSyntax
226+ ErrUnsupportedRangeHeader
227+ ErrLexerInvalidChar
228+ ErrLexerInvalidOperator
229+ ErrLexerInvalidLiteral
230+ ErrLexerInvalidIONLiteral
231+ ErrParseExpectedDatePart
232+ ErrParseExpectedKeyword
233+ ErrParseExpectedTokenType
234+ ErrParseExpected2TokenTypes
235+ ErrParseExpectedNumber
236+ ErrParseExpectedRightParenBuiltinFunctionCall
237+ ErrParseExpectedTypeName
238+ ErrParseExpectedWhenClause
239+ ErrParseUnsupportedToken
240+ ErrParseUnsupportedLiteralsGroupBy
241+ ErrParseExpectedMember
242+ ErrParseUnsupportedSelect
243+ ErrParseUnsupportedCase
244+ ErrParseUnsupportedCaseClause
245+ ErrParseUnsupportedAlias
246+ ErrParseUnsupportedSyntax
247+ ErrParseUnknownOperator
248+ ErrParseInvalidPathComponent
249+ ErrParseMissingIdentAfterAt
250+ ErrParseUnexpectedOperator
251+ ErrParseUnexpectedTerm
252+ ErrParseUnexpectedToken
253+ ErrParseUnexpectedKeyword
254+ ErrParseExpectedExpression
255+ ErrParseExpectedLeftParenAfterCast
256+ ErrParseExpectedLeftParenValueConstructor
257+ ErrParseExpectedLeftParenBuiltinFunctionCall
258+ ErrParseExpectedArgumentDelimiter
259+ ErrParseCastArity
260+ ErrParseInvalidTypeParam
261+ ErrParseEmptySelect
262+ ErrParseSelectMissingFrom
263+ ErrParseExpectedIdentForGroupName
264+ ErrParseExpectedIdentForAlias
265+ ErrParseUnsupportedCallWithStar
266+ ErrParseNonUnaryAgregateFunctionCall
267+ ErrParseMalformedJoin
268+ ErrParseExpectedIdentForAt
269+ ErrParseAsteriskIsNotAloneInSelectList
270+ ErrParseCannotMixSqbAndWildcardInSelectList
271+ ErrParseInvalidContextForWildcardInSelectList
272+ ErrIncorrectSQLFunctionArgumentType
273+ ErrValueParseFailure
274+ ErrEvaluatorInvalidArguments
275+ ErrIntegerOverflow
276+ ErrLikeInvalidInputs
277+ ErrCastFailed
278+ ErrInvalidCast
279+ ErrEvaluatorInvalidTimestampFormatPattern
280+ ErrEvaluatorInvalidTimestampFormatPatternSymbolForParsing
281+ ErrEvaluatorTimestampFormatPatternDuplicateFields
282+ ErrEvaluatorTimestampFormatPatternHourClockAmPmMismatch
283+ ErrEvaluatorUnterminatedTimestampFormatPatternToken
284+ ErrEvaluatorInvalidTimestampFormatPatternToken
285+ ErrEvaluatorInvalidTimestampFormatPatternSymbol
286+ ErrEvaluatorBindingDoesNotExist
287+ ErrInvalidColumnIndex
288+ ErrMissingHeaders
200289 )
201290
202291 // error code to APIError structure, these fields carry respective
@@ -803,6 +892,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{ · 600 unmodified lines
803892 Description: "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds",
804893 HTTPStatusCode: http.StatusBadRequest,
805894 },
895+
806896 // Generic Invalid-Request error. Should be used for response errors only for unlikely
807897 // corner case errors for which introducing new APIErrorCode is not worth it. LogIf()
808898 // should be used to log the error at the source of the error for debugging purposes.
@@ -851,6 +941,432 @@ var errorCodeResponse = map[APIErrorCode]APIError{ · 42 unmodified lines
851941 Description: "The continuation token provided is incorrect",
852942 HTTPStatusCode: http.StatusBadRequest,
853943 },
944+ //S3 Select API Errors
945+ ErrEmptyRequestBody: {
946+ Code: "EmptyRequestBody",
947+ Description: "Request body cannot be empty.",
948+ HTTPStatusCode: http.StatusBadRequest,
949+ },
950+ ErrUnsupportedFunction: {
951+ Code: "UnsupportedFunction",
952+ Description: "Encountered an unsupported SQL function.",
953+ HTTPStatusCode: http.StatusBadRequest,
954+ },
955+ ErrInvalidDataSource: {
956+ Code: "InvalidDataSource",
957+ Description: "Invalid data source type. Only CSV and JSON are supported at this time.",
958+ HTTPStatusCode: http.StatusBadRequest,
959+ },
960+ ErrInvalidExpressionType: {
961+ Code: "InvalidExpressionType",
962+ Description: "The ExpressionType is invalid. Only SQL expressions are supported at this time.",
963+ HTTPStatusCode: http.StatusBadRequest,
964+ },
965+ ErrBusy: {
966+ Code: "Busy",
967+ Description: "The service is unavailable. Please retry.",
968+ HTTPStatusCode: http.StatusServiceUnavailable,
969+ },
970+ ErrUnauthorizedAccess: {
971+ Code: "UnauthorizedAccess",
972+ Description: "You are not authorized to perform this operation",
973+ HTTPStatusCode: http.StatusUnauthorized,
974+ },
975+ ErrExpressionTooLong: {
976+ Code: "ExpressionTooLong",
977+ Description: "The SQL expression is too long: The maximum byte-length for the SQL expression is 256 KB.",
978+ HTTPStatusCode: http.StatusBadRequest,
979+ },
980+ ErrIllegalSQLFunctionArgument: {
981+ Code: "IllegalSqlFunctionArgument",
982+ Description: "Illegal argument was used in the SQL function.",
983+ HTTPStatusCode: http.StatusBadRequest,
984+ },
985+ ErrInvalidKeyPath: {
986+ Code: "InvalidKeyPath",
987+ Description: "Key path in the SQL expression is invalid.",
988+ HTTPStatusCode: http.StatusBadRequest,
989+ },
990+ ErrInvalidCompressionFormat: {
991+ Code: "InvalidCompressionFormat",
992+ Description: "The file is not in a supported compression format. Only GZIP is supported at this time.",
993+ HTTPStatusCode: http.StatusBadRequest,
994+ },
995+ ErrInvalidFileHeaderInfo: {
996+ Code: "InvalidFileHeaderInfo",
997+ Description: "The FileHeaderInfo is invalid. Only NONE, USE, and IGNORE are supported.",
998+ HTTPStatusCode: http.StatusBadRequest,
999+ },
1000+ ErrInvalidJSONType: {
1001+ Code: "InvalidJsonType",
1002+ Description: "The JsonType is invalid. Only DOCUMENT and LINES are supported at this time.",
1003+ HTTPStatusCode: http.StatusBadRequest,
1004+ },
1005+ ErrInvalidQuoteFields: {
1006+ Code: "InvalidQuoteFields",
1007+ Description: "The QuoteFields is invalid. Only ALWAYS and ASNEEDED are supported.",
1008+ HTTPStatusCode: http.StatusBadRequest,
1009+ },
1010+ ErrInvalidRequestParameter: {
1011+ Code: "InvalidRequestParameter",
1012+ Description: "The value of a parameter in SelectRequest element is invalid. Check the service API documentation and try again.",
1013+ HTTPStatusCode: http.StatusBadRequest,
1014+ },
1015+ ErrInvalidDataType: {
1016+ Code: "InvalidDataType",
1017+ Description: "The SQL expression contains an invalid data type.",
1018+ HTTPStatusCode: http.StatusBadRequest,
1019+ },
1020+ ErrInvalidTextEncoding: {
1021+ Code: "InvalidTextEncoding",
1022+ Description: "Invalid encoding type. Only UTF-8 encoding is supported at this time.",
1023+ HTTPStatusCode: http.StatusBadRequest,
1024+ },
1025+ ErrInvalidTableAlias: {
1026+ Code: "InvalidTableAlias",
1027+ Description: "The SQL expression contains an invalid table alias.",
1028+ HTTPStatusCode: http.StatusBadRequest,
1029+ },
1030+ ErrMissingRequiredParameter: {
1031+ Code: "MissingRequiredParameter",
1032+ Description: "The SelectRequest entity is missing a required parameter. Check the service documentation and try again.",
1033+ HTTPStatusCode: http.StatusBadRequest,
1034+ },
1035+ ErrObjectSerializationConflict: {
1036+ Code: "ObjectSerializationConflict",
1037+ Description: "The SelectRequest entity can only contain one of CSV or JSON. Check the service documentation and try again.",
1038+ HTTPStatusCode: http.StatusBadRequest,
1039+ },
1040+ ErrUnsupportedSQLOperation: {
1041+ Code: "UnsupportedSqlOperation",
1042+ Description: "Encountered an unsupported SQL operation.",
1043+ HTTPStatusCode: http.StatusBadRequest,
1044+ },
1045+ ErrUnsupportedSQLStructure: {
1046+ Code: "UnsupportedSqlStructure",
1047+ Description: "Encountered an unsupported SQL structure. Check the SQL Reference.",
1048+ HTTPStatusCode: http.StatusBadRequest,
1049+ },
1050+ ErrUnsupportedSyntax: {
1051+ Code: "UnsupportedSyntax",
1052+ Description: "Encountered invalid syntax.",
1053+ HTTPStatusCode: http.StatusBadRequest,
1054+ },
1055+ ErrUnsupportedRangeHeader: {
1056+ Code: "UnsupportedRangeHeader",
1057+ Description: "Range header is not supported for this operation.",
1058+ HTTPStatusCode: http.StatusBadRequest,
1059+ },
1060+ ErrLexerInvalidChar: {
1061+ Code: "LexerInvalidChar",
1062+ Description: "The SQL expression contains an invalid character.",
1063+ HTTPStatusCode: http.StatusBadRequest,
1064+ },
1065+ ErrLexerInvalidOperator: {
1066+ Code: "LexerInvalidOperator",
1067+ Description: "The SQL expression contains an invalid literal.",
1068+ HTTPStatusCode: http.StatusBadRequest,
1069+ },
1070+ ErrLexerInvalidLiteral: {
1071+ Code: "LexerInvalidLiteral",
1072+ Description: "The SQL expression contains an invalid operator.",
1073+ HTTPStatusCode: http.StatusBadRequest,
1074+ },
1075+ ErrLexerInvalidIONLiteral: {
1076+ Code: "LexerInvalidIONLiteral",
1077+ Description: "The SQL expression contains an invalid operator.",
1078+ HTTPStatusCode: http.StatusBadRequest,
1079+ },
1080+ ErrParseExpectedDatePart: {
1081+ Code: "ParseExpectedDatePart",
1082+ Description: "Did not find the expected date part in the SQL expression.",
1083+ HTTPStatusCode: http.StatusBadRequest,
1084+ },
1085+ ErrParseExpectedKeyword: {
1086+ Code: "ParseExpectedKeyword",
1087+ Description: "Did not find the expected keyword in the SQL expression.",
1088+ HTTPStatusCode: http.StatusBadRequest,
1089+ },
1090+ ErrParseExpectedTokenType: {
1091+ Code: "ParseExpectedTokenType",
1092+ Description: "Did not find the expected token in the SQL expression.",
1093+ HTTPStatusCode: http.StatusBadRequest,
1094+ },
1095+ ErrParseExpected2TokenTypes: {
1096+ Code: "ParseExpected2TokenTypes",
1097+ Description: "Did not find the expected token in the SQL expression.",
1098+ HTTPStatusCode: http.StatusBadRequest,
1099+ },
1100+ ErrParseExpectedNumber: {
1101+ Code: "ParseExpectedNumber",
1102+ Description: "Did not find the expected number in the SQL expression.",
1103+ HTTPStatusCode: http.StatusBadRequest,
1104+ },
1105+ ErrParseExpectedRightParenBuiltinFunctionCall: {
1106+ Code: "ParseExpectedRightParenBuiltinFunctionCall",
1107+ Description: "Did not find the expected right parenthesis character in the SQL expression.",
1108+ HTTPStatusCode: http.StatusBadRequest,
1109+ },
1110+ ErrParseExpectedTypeName: {
1111+ Code: "ParseExpectedTypeName",
1112+ Description: "Did not find the expected type name in the SQL expression.",
1113+ HTTPStatusCode: http.StatusBadRequest,
1114+ },
1115+ ErrParseExpectedWhenClause: {
1116+ Code: "ParseExpectedWhenClause",
1117+ Description: "Did not find the expected WHEN clause in the SQL expression. CASE is not supported.",
1118+ HTTPStatusCode: http.StatusBadRequest,
1119+ },
1120+ ErrParseUnsupportedToken: {
1121+ Code: "ParseUnsupportedToken",
1122+ Description: "The SQL expression contains an unsupported token.",
1123+ HTTPStatusCode: http.StatusBadRequest,
1124+ },
1125+ ErrParseUnsupportedLiteralsGroupBy: {
1126+ Code: "ParseUnsupportedLiteralsGroupBy",
1127+ Description: "The SQL expression contains an unsupported use of GROUP BY.",
1128+ HTTPStatusCode: http.StatusBadRequest,
1129+ },
1130+ ErrParseExpectedMember: {
1131+ Code: "ParseExpectedMember",
1132+ Description: "The SQL expression contains an unsupported use of MEMBER.",
1133+ HTTPStatusCode: http.StatusBadRequest,
1134+ },
1135+ ErrParseUnsupportedSelect: {
1136+ Code: "ParseUnsupportedSelect",
1137+ Description: "The SQL expression contains an unsupported use of SELECT.",
1138+ HTTPStatusCode: http.StatusBadRequest,
1139+ },
1140+ ErrParseUnsupportedCase: {
1141+ Code: "ParseUnsupportedCase",
1142+ Description: "The SQL expression contains an unsupported use of CASE.",
1143+ HTTPStatusCode: http.StatusBadRequest,
1144+ },
1145+ ErrParseUnsupportedCaseClause: {
1146+ Code: "ParseUnsupportedCaseClause",
1147+ Description: "The SQL expression contains an unsupported use of CASE.",
1148+ HTTPStatusCode: http.StatusBadRequest,
1149+ },
1150+ ErrParseUnsupportedAlias: {
1151+ Code: "ParseUnsupportedAlias",
1152+ Description: "The SQL expression contains an unsupported use of ALIAS.",
1153+ HTTPStatusCode: http.StatusBadRequest,
1154+ },
1155+ ErrParseUnsupportedSyntax: {
1156+ Code: "ParseUnsupportedSyntax",
1157+ Description: "The SQL expression contains unsupported syntax.",
1158+ HTTPStatusCode: http.StatusBadRequest,
1159+ },
1160+ ErrParseUnknownOperator: {
1161+ Code: "ParseUnknownOperator",
1162+ Description: "The SQL expression contains an invalid operator.",
1163+ HTTPStatusCode: http.StatusBadRequest,
1164+ },
1165+ ErrParseInvalidPathComponent: {
1166+ Code: "ParseInvalidPathComponent",
1167+ Description: "The SQL expression contains an invalid path component.",
1168+ HTTPStatusCode: http.StatusBadRequest,
1169+ },
1170+ ErrParseMissingIdentAfterAt: {
1171+ Code: "ParseMissingIdentAfterAt",
1172+ Description: "Did not find the expected identifier after the @ symbol in the SQL expression.",
1173+ HTTPStatusCode: http.StatusBadRequest,
1174+ },
1175+ ErrParseUnexpectedOperator: {
1176+ Code: "ParseUnexpectedOperator",
1177+ Description: "The SQL expression contains an unexpected operator.",
1178+ HTTPStatusCode: http.StatusBadRequest,
1179+ },
1180+ ErrParseUnexpectedTerm: {
1181+ Code: "ParseUnexpectedTerm",
1182+ Description: "The SQL expression contains an unexpected term.",
1183+ HTTPStatusCode: http.StatusBadRequest,
1184+ },
1185+ ErrParseUnexpectedToken: {
1186+ Code: "ParseUnexpectedToken",
1187+ Description: "The SQL expression contains an unexpected token.",
1188+ HTTPStatusCode: http.StatusBadRequest,
1189+ },
1190+ ErrParseUnexpectedKeyword: {
1191+ Code: "ParseUnexpectedKeyword",
1192+ Description: "The SQL expression contains an unexpected keyword.",
1193+ HTTPStatusCode: http.StatusBadRequest,
1194+ },
1195+ ErrParseExpectedExpression: {
1196+ Code: "ParseExpectedExpression",
1197+ Description: "Did not find the expected SQL expression.",
1198+ HTTPStatusCode: http.StatusBadRequest,
1199+ },
1200+ ErrParseExpectedLeftParenAfterCast: {
1201+ Code: "ParseExpectedLeftParenAfterCast",
1202+ Description: "Did not find expected the left parenthesis in the SQL expression.",
1203+ HTTPStatusCode: http.StatusBadRequest,
1204+ },
1205+ ErrParseExpectedLeftParenValueConstructor: {
1206+ Code: "ParseExpectedLeftParenValueConstructor",
1207+ Description: "Did not find expected the left parenthesis in the SQL expression.",
1208+ HTTPStatusCode: http.StatusBadRequest,
1209+ },
1210+ ErrParseExpectedLeftParenBuiltinFunctionCall: {
1211+ Code: "ParseExpectedLeftParenBuiltinFunctionCall",
1212+ Description: "Did not find the expected left parenthesis in the SQL expression.",
1213+ HTTPStatusCode: http.StatusBadRequest,
1214+ },
1215+ ErrParseExpectedArgumentDelimiter: {
1216+ Code: "ParseExpectedArgumentDelimiter",
1217+ Description: "Did not find the expected argument delimiter in the SQL expression.",
1218+ HTTPStatusCode: http.StatusBadRequest,
1219+ },
1220+ ErrParseCastArity: {
1221+ Code: "ParseCastArity",
1222+ Description: "The SQL expression CAST has incorrect arity.",
1223+ HTTPStatusCode: http.StatusBadRequest,
1224+ },
1225+ ErrParseInvalidTypeParam: {
1226+ Code: "ParseInvalidTypeParam",
1227+ Description: "The SQL expression contains an invalid parameter value.",
1228+ HTTPStatusCode: http.StatusBadRequest,
1229+ },
1230+ ErrParseEmptySelect: {
1231+ Code: "ParseEmptySelect",
1232+ Description: "The SQL expression contains an empty SELECT.",
1233+ HTTPStatusCode: http.StatusBadRequest,
1234+ },
1235+ ErrParseSelectMissingFrom: {
1236+ Code: "ParseSelectMissingFrom",
1237+ Description: "GROUP is not supported in the SQL expression.",
1238+ HTTPStatusCode: http.StatusBadRequest,
1239+ },
1240+ ErrParseExpectedIdentForGroupName: {
1241+ Code: "ParseExpectedIdentForGroupName",
1242+ Description: "GROUP is not supported in the SQL expression.",
1243+ HTTPStatusCode: http.StatusBadRequest,
1244+ },
1245+ ErrParseExpectedIdentForAlias: {
1246+ Code: "ParseExpectedIdentForAlias",
1247+ Description: "Did not find the expected identifier for the alias in the SQL expression.",
1248+ HTTPStatusCode: http.StatusBadRequest,
1249+ },
1250+ ErrParseUnsupportedCallWithStar: {
1251+ Code: "ParseUnsupportedCallWithStar",
1252+ Description: "Only COUNT with (*) as a parameter is supported in the SQL expression.",
1253+ HTTPStatusCode: http.StatusBadRequest,
1254+ },
1255+ ErrParseNonUnaryAgregateFunctionCall: {
1256+ Code: "ParseNonUnaryAgregateFunctionCall",
1257+ Description: "Only one argument is supported for aggregate functions in the SQL expression.",
1258+ HTTPStatusCode: http.StatusBadRequest,
1259+ },
1260+ ErrParseMalformedJoin: {
1261+ Code: "ParseMalformedJoin",
1262+ Description: "JOIN is not supported in the SQL expression.",
1263+ HTTPStatusCode: http.StatusBadRequest,
1264+ },
1265+ ErrParseExpectedIdentForAt: {
1266+ Code: "ParseExpectedIdentForAt",
1267+ Description: "Did not find the expected identifier for AT name in the SQL expression.",
1268+ HTTPStatusCode: http.StatusBadRequest,
1269+ },
1270+ ErrParseAsteriskIsNotAloneInSelectList: {
1271+ Code: "ParseAsteriskIsNotAloneInSelectList",
1272+ Description: "Other expressions are not allowed in the SELECT list when '*' is used without dot notation in the SQL expression.",
1273+ HTTPStatusCode: http.StatusBadRequest,
1274+ },
1275+ ErrParseCannotMixSqbAndWildcardInSelectList: {
1276+ Code: "ParseCannotMixSqbAndWildcardInSelectList",
1277+ Description: "Cannot mix [] and * in the same expression in a SELECT list in SQL expression.",
1278+ HTTPStatusCode: http.StatusBadRequest,
1279+ },
1280+ ErrParseInvalidContextForWildcardInSelectList: {
1281+ Code: "ParseInvalidContextForWildcardInSelectList",
1282+ Description: "Invalid use of * in SELECT list in the SQL expression.",
1283+ HTTPStatusCode: http.StatusBadRequest,
1284+ },
1285+ ErrIncorrectSQLFunctionArgumentType: {
1286+ Code: "IncorrectSqlFunctionArgumentType",
1287+ Description: "Incorrect type of arguments in function call in the SQL expression.",
1288+ HTTPStatusCode: http.StatusBadRequest,
1289+ },
1290+ ErrValueParseFailure: {
1291+ Code: "ValueParseFailure",
1292+ Description: "Time stamp parse failure in the SQL expression.",
1293+ HTTPStatusCode: http.StatusBadRequest,
1294+ },
1295+ ErrEvaluatorInvalidArguments: {
1296+ Code: "EvaluatorInvalidArguments",
1297+ Description: "Incorrect number of arguments in the function call in the SQL expression.",
1298+ HTTPStatusCode: http.StatusBadRequest,
1299+ },
1300+ ErrIntegerOverflow: {
1301+ Code: "IntegerOverflow",
1302+ Description: "Int overflow or underflow in the SQL expression.",
1303+ HTTPStatusCode: http.StatusBadRequest,
1304+ },
1305+ ErrLikeInvalidInputs: {
1306+ Code: "LikeInvalidInputs",
1307+ Description: "Invalid argument given to the LIKE clause in the SQL expression.",
1308+ HTTPStatusCode: http.StatusBadRequest,
1309+ },
1310+ ErrCastFailed: {
1311+ Code: "CastFailed",
1312+ Description: "Attempt to convert from one data type to another using CAST failed in the SQL expression.",
1313+ HTTPStatusCode: http.StatusBadRequest,
1314+ },
1315+ ErrInvalidCast: {
1316+ Code: "InvalidCast",
1317+ Description: "Attempt to convert from one data type to another using CAST failed in the SQL expression.",
1318+ HTTPStatusCode: http.StatusBadRequest,
1319+ },
1320+ ErrEvaluatorInvalidTimestampFormatPattern: {
1321+ Code: "EvaluatorInvalidTimestampFormatPattern",
1322+ Description: "Time stamp format pattern requires additional fields in the SQL expression.",
1323+ HTTPStatusCode: http.StatusBadRequest,
1324+ },
1325+ ErrEvaluatorInvalidTimestampFormatPatternSymbolForParsing: {
1326+ Code: "EvaluatorInvalidTimestampFormatPatternSymbolForParsing",
1327+ Description: "Time stamp format pattern contains a valid format symbol that cannot be applied to time stamp parsing in the SQL expression.",
1328+ HTTPStatusCode: http.StatusBadRequest,
1329+ },
1330+ ErrEvaluatorTimestampFormatPatternDuplicateFields: {
1331+ Code: "EvaluatorTimestampFormatPatternDuplicateFields",
1332+ Description: "Time stamp format pattern contains multiple format specifiers representing the time stamp field in the SQL expression.",
1333+ HTTPStatusCode: http.StatusBadRequest,
1334+ },
1335+ ErrEvaluatorTimestampFormatPatternHourClockAmPmMismatch: {
1336+ Code: "EvaluatorUnterminatedTimestampFormatPatternToken",
1337+ Description: "Time stamp format pattern contains unterminated token in the SQL expression.",
1338+ HTTPStatusCode: http.StatusBadRequest,
1339+ },
1340+ ErrEvaluatorUnterminatedTimestampFormatPatternToken: {
1341+ Code: "EvaluatorInvalidTimestampFormatPatternToken",
1342+ Description: "Time stamp format pattern contains an invalid token in the SQL expression.",
1343+ HTTPStatusCode: http.StatusBadRequest,
1344+ },
1345+ ErrEvaluatorInvalidTimestampFormatPatternToken: {
1346+ Code: "EvaluatorInvalidTimestampFormatPatternToken",
1347+ Description: "Time stamp format pattern contains an invalid token in the SQL expression.",
1348+ HTTPStatusCode: http.StatusBadRequest,
1349+ },
1350+ ErrEvaluatorInvalidTimestampFormatPatternSymbol: {
1351+ Code: "EvaluatorInvalidTimestampFormatPatternSymbol",
1352+ Description: "Time stamp format pattern contains an invalid symbol in the SQL expression.",
1353+ HTTPStatusCode: http.StatusBadRequest,
1354+ },
1355+ ErrInvalidColumnIndex: {
1356+ Code: "InvalidColumnIndex",
1357+ Description: "Column index in the SQL expression is invalid.",
1358+ HTTPStatusCode: http.StatusBadRequest,
1359+ },
1360+ ErrEvaluatorBindingDoesNotExist: {
1361+ Code: "ErrEvaluatorBindingDoesNotExist",
1362+ Description: "A column name or a path provided does not exist in the SQL expression",
1363+ HTTPStatusCode: http.StatusBadRequest,
1364+ },
1365+ ErrMissingHeaders: {
1366+ Code: "MissingHeaders",
1367+ Description: "Some headers in the query are missing from the file. Check the file and try again.",
1368+ HTTPStatusCode: http.StatusBadRequest,
1369+ },
8541370 // Add your error structure here.
8551371 }
8561372
@@ -900,6 +1416,169 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) { · 43 unmodified lines
9001416 case context.Canceled, context.DeadlineExceeded:
9011417 apiErr = ErrOperationTimedOut
9021418 }
1419+ switch err {
1420+ case s3select.ErrBusy:
1421+ apiErr = ErrBusy
1422+ case s3select.ErrUnauthorizedAccess:
1423+ apiErr = ErrUnauthorizedAccess
1424+ case s3select.ErrExpressionTooLong:
1425+ apiErr = ErrExpressionTooLong
1426+ case s3select.ErrIllegalSQLFunctionArgument:
1427+ apiErr = ErrIllegalSQLFunctionArgument
1428+ case s3select.ErrInvalidKeyPath:
1429+ apiErr = ErrInvalidKeyPath
1430+ case s3select.ErrInvalidCompressionFormat:
1431+ apiErr = ErrInvalidCompressionFormat
1432+ case s3select.ErrInvalidFileHeaderInfo:
1433+ apiErr = ErrInvalidFileHeaderInfo
1434+ case s3select.ErrInvalidJSONType:
1435+ apiErr = ErrInvalidJSONType
1436+ case s3select.ErrInvalidQuoteFields:
1437+ apiErr = ErrInvalidQuoteFields
1438+ case s3select.ErrInvalidRequestParameter:
1439+ apiErr = ErrInvalidRequestParameter
1440+ case s3select.ErrInvalidDataType:
1441+ apiErr = ErrInvalidDataType
1442+ case s3select.ErrInvalidTextEncoding:
1443+ apiErr = ErrInvalidTextEncoding
1444+ case s3select.ErrInvalidTableAlias:
1445+ apiErr = ErrInvalidTableAlias
1446+ case s3select.ErrMissingRequiredParameter:
1447+ apiErr = ErrMissingRequiredParameter
1448+ case s3select.ErrObjectSerializationConflict:
1449+ apiErr = ErrObjectSerializationConflict
1450+ case s3select.ErrUnsupportedSQLOperation:
1451+ apiErr = ErrUnsupportedSQLOperation
1452+ case s3select.ErrUnsupportedSQLStructure:
1453+ apiErr = ErrUnsupportedSQLStructure
1454+ case s3select.ErrUnsupportedSyntax:
1455+ apiErr = ErrUnsupportedSyntax
1456+ case s3select.ErrUnsupportedRangeHeader:
1457+ apiErr = ErrUnsupportedRangeHeader
1458+ case s3select.ErrLexerInvalidChar:
1459+ apiErr = ErrLexerInvalidChar
1460+ case s3select.ErrLexerInvalidOperator:
1461+ apiErr = ErrLexerInvalidOperator
1462+ case s3select.ErrLexerInvalidLiteral:
1463+ apiErr = ErrLexerInvalidLiteral
1464+ case s3select.ErrLexerInvalidIONLiteral:
1465+ apiErr = ErrLexerInvalidIONLiteral
1466+ case s3select.ErrParseExpectedDatePart:
1467+ apiErr = ErrParseExpectedDatePart
1468+ case s3select.ErrParseExpectedKeyword:
1469+ apiErr = ErrParseExpectedKeyword
1470+ case s3select.ErrParseExpectedTokenType:
1471+ apiErr = ErrParseExpectedTokenType
1472+ case s3select.ErrParseExpected2TokenTypes:
1473+ apiErr = ErrParseExpected2TokenTypes
1474+ case s3select.ErrParseExpectedNumber:
1475+ apiErr = ErrParseExpectedNumber
1476+ case s3select.ErrParseExpectedRightParenBuiltinFunctionCall:
1477+ apiErr = ErrParseExpectedRightParenBuiltinFunctionCall
1478+ case s3select.ErrParseExpectedTypeName:
1479+ apiErr = ErrParseExpectedTypeName
1480+ case s3select.ErrParseExpectedWhenClause:
1481+ apiErr = ErrParseExpectedWhenClause
1482+ case s3select.ErrParseUnsupportedToken:
1483+ apiErr = ErrParseUnsupportedToken
1484+ case s3select.ErrParseUnsupportedLiteralsGroupBy:
1485+ apiErr = ErrParseUnsupportedLiteralsGroupBy
1486+ case s3select.ErrParseExpectedMember:
1487+ apiErr = ErrParseExpectedMember
1488+ case s3select.ErrParseUnsupportedSelect:
1489+ apiErr = ErrParseUnsupportedSelect
1490+ case s3select.ErrParseUnsupportedCase:
1491+ apiErr = ErrParseUnsupportedCase
1492+ case s3select.ErrParseUnsupportedCaseClause:
1493+ apiErr = ErrParseUnsupportedCaseClause
1494+ case s3select.ErrParseUnsupportedAlias:
1495+ apiErr = ErrParseUnsupportedAlias
1496+ case s3select.ErrParseUnsupportedSyntax:
1497+ apiErr = ErrParseUnsupportedSyntax
1498+ case s3select.ErrParseUnknownOperator:
1499+ apiErr = ErrParseUnknownOperator
1500+ case s3select.ErrParseInvalidPathComponent:
1501+ apiErr = ErrParseInvalidPathComponent
1502+ case s3select.ErrParseMissingIdentAfterAt:
1503+ apiErr = ErrParseMissingIdentAfterAt
1504+ case s3select.ErrParseUnexpectedOperator:
1505+ apiErr = ErrParseUnexpectedOperator
1506+ case s3select.ErrParseUnexpectedTerm:
1507+ apiErr = ErrParseUnexpectedTerm
1508+ case s3select.ErrParseUnexpectedToken:
1509+ apiErr = ErrParseUnexpectedToken
1510+ case s3select.ErrParseUnexpectedKeyword:
1511+ apiErr = ErrParseUnexpectedKeyword
1512+ case s3select.ErrParseExpectedExpression:
1513+ apiErr = ErrParseExpectedExpression
1514+ case s3select.ErrParseExpectedLeftParenAfterCast:
1515+ apiErr = ErrParseExpectedLeftParenAfterCast
1516+ case s3select.ErrParseExpectedLeftParenValueConstructor:
1517+ apiErr = ErrParseExpectedLeftParenValueConstructor
1518+ case s3select.ErrParseExpectedLeftParenBuiltinFunctionCall:
1519+ apiErr = ErrParseExpectedLeftParenBuiltinFunctionCall
1520+ case s3select.ErrParseExpectedArgumentDelimiter:
1521+ apiErr = ErrParseExpectedArgumentDelimiter
1522+ case s3select.ErrParseCastArity:
1523+ apiErr = ErrParseCastArity
1524+ case s3select.ErrParseInvalidTypeParam:
1525+ apiErr = ErrParseInvalidTypeParam
1526+ case s3select.ErrParseEmptySelect:
1527+ apiErr = ErrParseEmptySelect
1528+ case s3select.ErrParseSelectMissingFrom:
1529+ apiErr = ErrParseSelectMissingFrom
1530+ case s3select.ErrParseExpectedIdentForGroupName:
1531+ apiErr = ErrParseExpectedIdentForGroupName
1532+ case s3select.ErrParseExpectedIdentForAlias:
1533+ apiErr = ErrParseExpectedIdentForAlias
1534+ case s3select.ErrParseUnsupportedCallWithStar:
1535+ apiErr = ErrParseUnsupportedCallWithStar
1536+ case s3select.ErrParseNonUnaryAgregateFunctionCall:
1537+ apiErr = ErrParseNonUnaryAgregateFunctionCall
1538+ case s3select.ErrParseMalformedJoin:
1539+ apiErr = ErrParseMalformedJoin
1540+ case s3select.ErrParseExpectedIdentForAt:
1541+ apiErr = ErrParseExpectedIdentForAt
1542+ case s3select.ErrParseAsteriskIsNotAloneInSelectList:
1543+ apiErr = ErrParseAsteriskIsNotAloneInSelectList
1544+ case s3select.ErrParseCannotMixSqbAndWildcardInSelectList:
1545+ apiErr = ErrParseCannotMixSqbAndWildcardInSelectList
1546+ case s3select.ErrParseInvalidContextForWildcardInSelectList:
1547+ apiErr = ErrParseInvalidContextForWildcardInSelectList
1548+ case s3select.ErrIncorrectSQLFunctionArgumentType:
1549+ apiErr = ErrIncorrectSQLFunctionArgumentType
1550+ case s3select.ErrValueParseFailure:
1551+ apiErr = ErrValueParseFailure
1552+ case s3select.ErrIntegerOverflow:
1553+ apiErr = ErrIntegerOverflow
1554+ case s3select.ErrLikeInvalidInputs:
1555+ apiErr = ErrLikeInvalidInputs
1556+ case s3select.ErrCastFailed:
1557+ apiErr = ErrCastFailed
1558+ case s3select.ErrInvalidCast:
1559+ apiErr = ErrInvalidCast
1560+ case s3select.ErrEvaluatorInvalidTimestampFormatPattern:
1561+ apiErr = ErrEvaluatorInvalidTimestampFormatPattern
1562+ case s3select.ErrEvaluatorInvalidTimestampFormatPatternSymbolForParsing:
1563+ apiErr = ErrEvaluatorInvalidTimestampFormatPatternSymbolForParsing
1564+ case s3select.ErrEvaluatorTimestampFormatPatternDuplicateFields:
1565+ apiErr = ErrEvaluatorTimestampFormatPatternDuplicateFields
1566+ case s3select.ErrEvaluatorTimestampFormatPatternHourClockAmPmMismatch:
1567+ apiErr = ErrEvaluatorTimestampFormatPatternHourClockAmPmMismatch
1568+ case s3select.ErrEvaluatorUnterminatedTimestampFormatPatternToken:
1569+ apiErr = ErrEvaluatorUnterminatedTimestampFormatPatternToken
1570+ case s3select.ErrEvaluatorInvalidTimestampFormatPatternToken:
1571+ apiErr = ErrEvaluatorInvalidTimestampFormatPatternToken
1572+ case s3select.ErrEvaluatorInvalidTimestampFormatPatternSymbol:
1573+ apiErr = ErrEvaluatorInvalidTimestampFormatPatternSymbol
1574+ case s3select.ErrInvalidColumnIndex:
1575+ apiErr = ErrInvalidColumnIndex
1576+ case s3select.ErrEvaluatorBindingDoesNotExist:
1577+ apiErr = ErrEvaluatorBindingDoesNotExist
1578+ case s3select.ErrMissingHeaders:
1579+ apiErr = ErrMissingHeaders
1580+
1581+ }
9031582
9041583 if apiErr != ErrNone {
9051584 // If there was a match in the above switch case.
cmd/api-router.go+2−0
@@ -62,6 +62,8 @@ func registerAPIRouter(router *mux.Router) { · 61 unmodified lines
6262 bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
6363 // GetObjectACL - this is a dummy call.
6464 bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectACLHandler)).Queries("acl", "")
65+ // SelectObjectContent
66+ bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.SelectObjectContentHandler)).Queries("select", "").Queries("select-type", "2")
6567 // GetObject
6668 bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectHandler))
6769 // CopyObject
cmd/generic-handlers.go+0−1
@@ -475,7 +475,6 @@ var notimplementedObjectResourceNames = map[string]bool{ · 474 unmodified lines
475475 "policy": true,
476476 "tagging": true,
477477 "restore": true,
478− "select": true,
479478 }
480479
481480 // Resource handler ServeHTTP() wrapper
cmd/object-handlers.go+187−0
@@ -30,6 +30,7 @@ import ( · 29 unmodified lines
3030 "net/url"
3131 "sort"
3232 "strconv"
33+ "strings"
3334
3435 "github.com/gorilla/mux"
3536 miniogo "github.com/minio/minio-go"
@@ -40,6 +41,7 @@ import ( · 4 unmodified lines
4041 "github.com/minio/minio/pkg/hash"
4142 "github.com/minio/minio/pkg/ioutil"
4243 "github.com/minio/minio/pkg/policy"
44+ "github.com/minio/minio/pkg/s3select"
4345 sha256 "github.com/minio/sha256-simd"
4446 "github.com/minio/sio"
4547 )
@@ -63,6 +65,191 @@ func setHeadGetRespHeaders(w http.ResponseWriter, reqParams url.Values) { · 17 unmodified lines
6365 }
6466 }
6567
68+// SelectObjectContentHandler - GET Object?select
69+// ----------
70+// This implementation of the GET operation retrieves object content based
71+// on an SQL expression. In the request, along with the sql expression, you must
72+// also specify a data serialization format (JSON, CSV) of the object.
73+func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r *http.Request) {
74+ ctx := newContext(r, w, "SelectObject")
75+ var object, bucket string
76+ vars := mux.Vars(r)
77+ bucket = vars["bucket"]
78+ object = vars["object"]
79+
80+ // Fetch object stat info.
81+ objectAPI := api.ObjectAPI()
82+ if objectAPI == nil {
83+ writeErrorResponse(w, ErrServerNotInitialized, r.URL)
84+ return
85+ }
86+
87+ getObjectInfo := objectAPI.GetObjectInfo
88+ if api.CacheAPI() != nil {
89+ getObjectInfo = api.CacheAPI().GetObjectInfo
90+ }
91+
92+ if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectAction, bucket, object); s3Error != ErrNone {
93+ if getRequestAuthType(r) == authTypeAnonymous {
94+ // As per "Permission" section in
95+ // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html If
96+ // the object you request does not exist, the error Amazon S3 returns
97+ // depends on whether you also have the s3:ListBucket permission. * If you
98+ // have the s3:ListBucket permission on the bucket, Amazon S3 will return
99+ // an HTTP status code 404 ("no such key") error. * if you don’t have the
100+ // s3:ListBucket permission, Amazon S3 will return an HTTP status code 403
101+ // ("access denied") error.`
102+ if globalPolicySys.IsAllowed(policy.Args{
103+ Action: policy.ListBucketAction,
104+ BucketName: bucket,
105+ ConditionValues: getConditionValues(r, ""),
106+ IsOwner: false,
107+ }) {
108+ _, err := getObjectInfo(ctx, bucket, object)
109+ if toAPIErrorCode(err) == ErrNoSuchKey {
110+ s3Error = ErrNoSuchKey
111+ }
112+ }
113+ }
114+ writeErrorResponse(w, s3Error, r.URL)
115+ return
116+ }
117+ if r.ContentLength <= 0 {
118+ writeErrorResponse(w, ErrEmptyRequestBody, r.URL)
119+ return
120+ }
121+ var selectReq ObjectSelectRequest
122+ if err := xmlDecoder(r.Body, &selectReq, r.ContentLength); err != nil {
123+ fmt.Println(err)
124+ writeErrorResponse(w, ErrMalformedXML, r.URL)
125+ return
126+ }
127+
128+ objInfo, err := getObjectInfo(ctx, bucket, object)
129+ if err != nil {
130+ writeErrorResponse(w, toAPIErrorCode(err), r.URL)
131+ return
132+ }
133+ // Get request range.
134+ rangeHeader := r.Header.Get("Range")
135+ if rangeHeader != "" {
136+ writeErrorResponse(w, ErrUnsupportedRangeHeader, r.URL)
137+ return
138+ }
139+
140+ if selectReq.InputSerialization.CompressionType == SelectCompressionGZIP {
141+ if !strings.Contains(objInfo.ContentType, "gzip") {
142+ writeErrorResponse(w, ErrInvalidDataSource, r.URL)
143+ return
144+ }
145+ }
146+ if selectReq.InputSerialization.CompressionType == SelectCompressionBZIP {
147+ if !strings.Contains(objInfo.ContentType, "bzip") {
148+ writeErrorResponse(w, ErrInvalidDataSource, r.URL)
149+ return
150+ }
151+ }
152+ if selectReq.InputSerialization.CompressionType == SelectCompressionNONE ||
153+ selectReq.InputSerialization.CompressionType == "" {
154+ selectReq.InputSerialization.CompressionType = SelectCompressionNONE
155+ if !strings.Contains(objInfo.ContentType, "text/csv") {
156+ writeErrorResponse(w, ErrInvalidDataSource, r.URL)
157+ return
158+ }
159+ }
160+ if !strings.EqualFold(string(selectReq.ExpressionType), "SQL") {
161+ writeErrorResponse(w, ErrInvalidExpressionType, r.URL)
162+ return
163+ }
164+ if len(selectReq.Expression) >= (256 * 1000) {
165+ writeErrorResponse(w, ErrExpressionTooLong, r.URL)
166+ }
167+ if selectReq.InputSerialization.CSV.FileHeaderInfo != CSVFileHeaderInfoUse &&
168+ selectReq.InputSerialization.CSV.FileHeaderInfo != CSVFileHeaderInfoNone &&
169+ selectReq.InputSerialization.CSV.FileHeaderInfo != CSVFileHeaderInfoIgnore &&
170+ selectReq.InputSerialization.CSV.FileHeaderInfo != "" {
171+ writeErrorResponse(w, ErrInvalidFileHeaderInfo, r.URL)
172+ }
173+ if selectReq.OutputSerialization.CSV.QuoteFields != CSVQuoteFieldsAlways &&
174+ selectReq.OutputSerialization.CSV.QuoteFields != CSVQuoteFieldsAsNeeded &&
175+ selectReq.OutputSerialization.CSV.QuoteFields != "" {
176+ writeErrorResponse(w, ErrInvalidQuoteFields, r.URL)
177+ }
178+
179+ getObject := objectAPI.GetObject
180+ if api.CacheAPI() != nil && !hasSSECustomerHeader(r.Header) {
181+ getObject = api.CacheAPI().GetObject
182+ }
183+
184+ reader, pipewriter := io.Pipe()
185+
186+ // Get the object.
187+ var startOffset int64
188+ length := objInfo.Size
189+
190+ var writer io.Writer
191+ writer = pipewriter
192+ if objectAPI.IsEncryptionSupported() {
193+ if hasSSECustomerHeader(r.Header) {
194+ // Response writer should be limited early on for decryption upto required length,
195+ // additionally also skipping mod(offset)64KiB boundaries.
196+ writer = ioutil.LimitedWriter(writer, startOffset%(64*1024), length)
197+
198+ writer, startOffset, length, err = DecryptBlocksRequest(writer, r, bucket,
199+ object, startOffset, length, objInfo, false)
200+ if err != nil {
201+ writeErrorResponse(w, toAPIErrorCode(err), r.URL)
202+ return
203+ }
204+ }
205+ }
206+ go func() {
207+ defer reader.Close()
208+ if gerr := getObject(ctx, bucket, object, 0, objInfo.Size, writer,
209+ objInfo.ETag); gerr != nil {
210+ pipewriter.CloseWithError(gerr)
211+ return
212+ }
213+ pipewriter.Close() // Close writer explicitly signaling we wrote all data.
214+ }()
215+
216+ //s3select //Options
217+ if selectReq.OutputSerialization.CSV.FieldDelimiter == "" {
218+ selectReq.OutputSerialization.CSV.FieldDelimiter = ","
219+ }
220+ if selectReq.InputSerialization.CSV.FileHeaderInfo == "" {
221+ selectReq.InputSerialization.CSV.FileHeaderInfo = CSVFileHeaderInfoNone
222+ }
223+ if selectReq.InputSerialization.CSV != nil {
224+ options := &s3select.Options{
225+ HasHeader: selectReq.InputSerialization.CSV.FileHeaderInfo != CSVFileHeaderInfoNone,
226+ FieldDelimiter: selectReq.InputSerialization.CSV.FieldDelimiter,
227+ Comments: selectReq.InputSerialization.CSV.Comments,
228+ Name: "S3Object", // Default table name for all objects
229+ ReadFrom: reader,
230+ Compressed: string(selectReq.InputSerialization.CompressionType),
231+ Expression: selectReq.Expression,
232+ OutputFieldDelimiter: selectReq.OutputSerialization.CSV.FieldDelimiter,
233+ StreamSize: objInfo.Size,
234+ HeaderOpt: selectReq.InputSerialization.CSV.FileHeaderInfo == CSVFileHeaderInfoUse,
235+ }
236+ s3s, err := s3select.NewInput(options)
237+ if err != nil {
238+ writeErrorResponse(w, toAPIErrorCode(err), r.URL)
239+ return
240+ }
241+ _, _, _, _, _, _, err = s3s.ParseSelect(selectReq.Expression)
242+ if err != nil {
243+ writeErrorResponse(w, toAPIErrorCode(err), r.URL)
244+ return
245+ }
246+ if err := s3s.Execute(w); err != nil {
247+ logger.LogIf(ctx, err)
248+ }
249+ return
250+ }
251+}
252+
66253 // GetObjectHandler - GET Object
67254 // ----------
68255 // This implementation of the GET operation retrieves object. To use GET,
docs/select/README.md+98−0
@@ -0,0 +1,98 @@
1+# How to use Minio S3 Select [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io)
2+
3+This document explains current limitations of the Minio S3 Select support.
4+
5+## 1. Features to be implemented
6+1). JSON documents as supported Objects
7+
8+2). CAST expression
9+
10+3). Date Functions
11+
12+4). Returning types other than float from aggregation queries.
13+
14+5). Bracket and Reversal Notation with SQL Like operator.
15+
16+6). SUBSTRING currently is not supported and TRIM only works with default arguments of trim leading and trailing spaces
17+
18+## 2. Sample Usage with AWS Boto Client
19+```python
20+import boto3
21+from botocore.client import Config
22+import os
23+s3 = boto3.resource('s3',
24+ endpoint_url='ENDPOINT',
25+ aws_access_key_id='ACCESSKEY',
26+ aws_secret_access_key='SECRETKEY',
27+ config=Config(signature_version='s3v4'),
28+ region_name='us-east-1')
29+s3_client = s3.meta.client
30+
31+r = s3_client.select_object_content(
32+ Bucket='myBucket',
33+ Key='myKey',
34+ ExpressionType='SQL',
35+ Expression = "SELECT * FROM S3OBJECT AS A",
36+ InputSerialization = {'CSV': {"FieldDelimiter": ",","FileHeaderInfo":"USE"}},
37+ OutputSerialization = {'CSV': {}},
38+ )
39+```
40+## 3. Sample Usage with Minio-Go Client
41+
42+```go
43+ // Initialize minio client object.
44+ minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, useSSL)
45+ if err != nil {
46+ log.Fatalln(err)
47+ }
48+ input := minio.SelectObjectInput{
49+ RecordDelimiter: "\n",
50+ FieldDelimiter: ",",
51+ FileHeaderInfo: minio.CSVFileHeaderInfoUse,
52+ }
53+ output := minio.SelectObjectOutput{
54+ RecordDelimiter: "\n",
55+ FieldDelimiter: ",",
56+ }
57+ opts := minio.SelectObjectOptions{
58+ Type: minio.SelectObjectTypeCSV,
59+ Input: input,
60+ Output: output,
61+ }
62+ myReader, err := minioClient.SelectObjectContent(ctx, "sqlselectapi", "player.csv", "Select * from S3OBJECT WHERE last_name = 'James'", opts)
63+ if err != nil {
64+ fmt.Println(err)
65+ return
66+ }
67+ defer myReader.Close()
68+
69+ results, resultWriter := io.Pipe()
70+ go func() {
71+ defer resultWriter.Close()
72+ for event := range myReader.Events() {
73+ switch e := event.(type) {
74+ case *minio.RecordEvent:
75+ resultWriter.Write(e.Payload)
76+ case *minio.ProgressEvent:
77+ fmt.Println("Progress")
78+ case *minio.StatEvent:
79+ fmt.Println(string(e.Payload))
80+ case *minio.EndEvent:
81+ fmt.Println("Ended")
82+ return
83+ }
84+ }
85+ }()
86+ resReader := csv.NewReader(results)
87+ for {
88+ record, err := resReader.Read()
89+ if err == io.EOF {
90+ break
91+ }
92+ // Print out the records
93+ fmt.Println(record)
94+ }
95+ if err := myReader.Err(); err != nil {
96+ fmt.Println(err)
97+ }
98+```
pkg/s3select/errors.go+486−0
@@ -0,0 +1,486 @@
1+/*
2+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
3+ *
4+ * Licensed under the Apache License, Version 2.0 (the "License");
5+ * you may not use this file except in compliance with the License.
6+ * You may obtain a copy of the License at
7+ *
8+ * http://www.apache.org/licenses/LICENSE-2.0
9+ *
10+ * Unless required by applicable law or agreed to in writing, software
11+ * distributed under the License is distributed on an "AS IS" BASIS,
12+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+ * See the License for the specific language governing permissions and
14+ * limitations under the License.
15+ */
16+
17+package s3select
18+
19+import "errors"
20+
21+//S3 errors below
22+
23+// ErrBusy is an error if the service is too busy.
24+var ErrBusy = errors.New("The service is unavailable. Please retry")
25+
26+// ErrUnauthorizedAccess is an error if you lack the appropriate credentials to
27+// access the object.
28+var ErrUnauthorizedAccess = errors.New("You are not authorized to perform this operation")
29+
30+// ErrExpressionTooLong is an error if your SQL expression too long for
31+// processing.
32+var ErrExpressionTooLong = errors.New("The SQL expression is too long: The maximum byte-length for the SQL expression is 256 KB")
33+
34+// ErrIllegalSQLFunctionArgument is an error if you provide an illegal argument
35+// in the SQL function.
36+var ErrIllegalSQLFunctionArgument = errors.New("Illegal argument was used in the SQL function")
37+
38+// ErrInvalidColumnIndex is an error if you provide a column index which is not
39+// valid.
40+var ErrInvalidColumnIndex = errors.New("Column index in the SQL expression is invalid")
41+
42+// ErrInvalidKeyPath is an error if you provide a key in the SQL expression that
43+// is invalid.
44+var ErrInvalidKeyPath = errors.New("Key path in the SQL expression is invalid")
45+
46+// ErrColumnTooLong is an error if your query results in a column that is
47+// greater than the max amount of characters per column of 1mb
48+var ErrColumnTooLong = errors.New("The length of a column in the result is greater than maxCharsPerColumn of 1 MB")
49+
50+// ErrOverMaxColumn is an error if the number of columns from the resulting
51+// query is greater than 1Mb.
52+var ErrOverMaxColumn = errors.New("The number of columns in the result is greater than maxColumnNumber of 1 MB")
53+
54+// ErrOverMaxRecordSize is an error if the length of a record in the result is
55+// greater than 1 Mb.
56+var ErrOverMaxRecordSize = errors.New("The length of a record in the result is greater than maxCharsPerRecord of 1 MB")
57+
58+// ErrMissingHeaders is an error if some of the headers that are requested in
59+// the Select Query are not present in the file.
60+var ErrMissingHeaders = errors.New("Some headers in the query are missing from the file. Check the file and try again")
61+
62+// ErrInvalidCompressionFormat is an error if an unsupported compression type is
63+// utilized with the select object query.
64+var ErrInvalidCompressionFormat = errors.New("The file is not in a supported compression format. Only GZIP is supported at this time")
65+
66+// ErrTruncatedInput is an error if the object is not compressed properly and an
67+// error occurs during decompression.
68+var ErrTruncatedInput = errors.New("Object decompression failed. Check that the object is properly compressed using the format specified in the request")
69+
70+// ErrInvalidFileHeaderInfo is an error if the argument provided to the
71+// FileHeader Argument is incorrect.
72+var ErrInvalidFileHeaderInfo = errors.New("The FileHeaderInfo is invalid. Only NONE, USE, and IGNORE are supported")
73+
74+// ErrInvalidJSONType is an error if the json format provided as an argument is
75+// invalid.
76+var ErrInvalidJSONType = errors.New("The JsonType is invalid. Only DOCUMENT and LINES are supported at this time")
77+
78+// ErrInvalidQuoteFields is an error if the arguments provided to the
79+// QuoteFields options are not valid.
80+var ErrInvalidQuoteFields = errors.New("The QuoteFields is invalid. Only ALWAYS and ASNEEDED are supported")
81+
82+// ErrInvalidRequestParameter is an error if the value of a parameter in the
83+// request element is not valid.
84+var ErrInvalidRequestParameter = errors.New("The value of a parameter in Request element is invalid. Check the service API documentation and try again")
85+
86+// ErrCSVParsingError is an error if the CSV presents an error while being
87+// parsed.
88+var ErrCSVParsingError = errors.New("Encountered an Error parsing the CSV file. Check the file and try again")
89+
90+// ErrJSONParsingError is an error if while parsing the JSON an error arises.
91+var ErrJSONParsingError = errors.New("Encountered an error parsing the JSON file. Check the file and try again")
92+
93+// ErrExternalEvalException is an error that arises if the query can not be
94+// evaluated.
95+var ErrExternalEvalException = errors.New("The query cannot be evaluated. Check the file and try again")
96+
97+// ErrInvalidDataType is an error that occurs if the SQL expression contains an
98+// invalid data type.
99+var ErrInvalidDataType = errors.New("The SQL expression contains an invalid data type")
100+
101+// ErrUnrecognizedFormatException is an error that arises if there is an invalid
102+// record type.
103+var ErrUnrecognizedFormatException = errors.New("Encountered an invalid record type")
104+
105+// ErrInvalidTextEncoding is an error if the text encoding is not valid.
106+var ErrInvalidTextEncoding = errors.New("Invalid encoding type. Only UTF-8 encoding is supported at this time")
107+
108+// ErrInvalidTableAlias is an error that arises if the table alias provided in
109+// the SQL expression is invalid.
110+var ErrInvalidTableAlias = errors.New("The SQL expression contains an invalid table alias")
111+
112+// ErrMultipleDataSourcesUnsupported is an error that arises if multiple data
113+// sources are provided.
114+var ErrMultipleDataSourcesUnsupported = errors.New("Multiple data sources are not supported")
115+
116+// ErrMissingRequiredParameter is an error that arises if a required argument
117+// is omitted from the Request.
118+var ErrMissingRequiredParameter = errors.New("The Request entity is missing a required parameter. Check the service documentation and try again")
119+
120+// ErrObjectSerializationConflict is an error that arises if an unsupported
121+// output seralization is provided.
122+var ErrObjectSerializationConflict = errors.New("The Request entity can only contain one of CSV or JSON. Check the service documentation and try again")
123+
124+// ErrUnsupportedSQLOperation is an error that arises if an unsupported SQL
125+// operation is used.
126+var ErrUnsupportedSQLOperation = errors.New("Encountered an unsupported SQL operation")
127+
128+// ErrUnsupportedSQLStructure is an error that occurs if an unsupported SQL
129+// structure is used.
130+var ErrUnsupportedSQLStructure = errors.New("Encountered an unsupported SQL structure. Check the SQL Reference")
131+
132+// ErrUnsupportedStorageClass is an error that occurs if an invalid storace
133+// class is present.
134+var ErrUnsupportedStorageClass = errors.New("Encountered an invalid storage class. Only STANDARD, STANDARD_IA, and ONEZONE_IA storage classes are supported at this time")
135+
136+// ErrUnsupportedSyntax is an error that occurs if invalid syntax is present in
137+// the query.
138+var ErrUnsupportedSyntax = errors.New("Encountered invalid syntax")
139+
140+// ErrUnsupportedRangeHeader is an error that occurs if a range header is
141+// provided.
142+var ErrUnsupportedRangeHeader = errors.New("Range header is not supported for this operation")
143+
144+// ErrLexerInvalidChar is an error that occurs if the SQL expression contains an
145+// invalid character.
146+var ErrLexerInvalidChar = errors.New("The SQL expression contains an invalid character")
147+
148+// ErrLexerInvalidOperator is an error that occurs if an invalid operator is
149+// used.
150+var ErrLexerInvalidOperator = errors.New("The SQL expression contains an invalid operator")
151+
152+// ErrLexerInvalidLiteral is an error that occurs if an invalid literal is used.
153+var ErrLexerInvalidLiteral = errors.New("The SQL expression contains an invalid literal")
154+
155+// ErrLexerInvalidIONLiteral is an error that occurs if an invalid operator is
156+// used
157+var ErrLexerInvalidIONLiteral = errors.New("The SQL expression contains an invalid operator")
158+
159+// ErrParseExpectedDatePart is an error that occurs if the date part is not
160+// found in the SQL expression.
161+var ErrParseExpectedDatePart = errors.New("Did not find the expected date part in the SQL expression")
162+
163+// ErrParseExpectedKeyword is an error that occurs if the expected keyword was
164+// not found in the expression.
165+var ErrParseExpectedKeyword = errors.New("Did not find the expected keyword in the SQL expression")
166+
167+// ErrParseExpectedTokenType is an error that occurs if the expected token is
168+// not found in the SQL expression.
169+var ErrParseExpectedTokenType = errors.New("Did not find the expected token in the SQL expression")
170+
171+// ErrParseExpected2TokenTypes is an error that occurs if 2 token types are not
172+// found.
173+var ErrParseExpected2TokenTypes = errors.New("Did not find the expected token in the SQL expression")
174+
175+// ErrParseExpectedNumber is an error that occurs if a number is expected but
176+// not found in the expression.
177+var ErrParseExpectedNumber = errors.New("Did not find the expected number in the SQL expression")
178+
179+// ErrParseExpectedRightParenBuiltinFunctionCall is an error that occurs if a
180+// right parenthesis is missing.
181+var ErrParseExpectedRightParenBuiltinFunctionCall = errors.New("Did not find the expected right parenthesis character in the SQL expression")
182+
183+// ErrParseExpectedTypeName is an error that occurs if a type name is expected
184+// but not found.
185+var ErrParseExpectedTypeName = errors.New("Did not find the expected type name in the SQL expression")
186+
187+// ErrParseExpectedWhenClause is an error that occurs if a When clause is
188+// expected but not found.
189+var ErrParseExpectedWhenClause = errors.New("Did not find the expected WHEN clause in the SQL expression. CASE is not supported")
190+
191+// ErrParseUnsupportedToken is an error that occurs if the SQL expression
192+// contains an unsupported token.
193+var ErrParseUnsupportedToken = errors.New("The SQL expression contains an unsupported token")
194+
195+// ErrParseUnsupportedLiteralsGroupBy is an error that occurs if the SQL
196+// expression has an unsupported use of Group By.
197+var ErrParseUnsupportedLiteralsGroupBy = errors.New("The SQL expression contains an unsupported use of GROUP BY")
198+
199+// ErrParseExpectedMember is an error that occurs if there is an unsupported use
200+// of member in the SQL expression.
201+var ErrParseExpectedMember = errors.New("The SQL expression contains an unsupported use of MEMBER")
202+
203+// ErrParseUnsupportedSelect is an error that occurs if there is an unsupported
204+// use of Select.
205+var ErrParseUnsupportedSelect = errors.New("The SQL expression contains an unsupported use of SELECT")
206+
207+// ErrParseUnsupportedCase is an error that occurs if there is an unsupported
208+// use of case.
209+var ErrParseUnsupportedCase = errors.New("The SQL expression contains an unsupported use of CASE")
210+
211+// ErrParseUnsupportedCaseClause is an error that occurs if there is an
212+// unsupported use of case.
213+var ErrParseUnsupportedCaseClause = errors.New("The SQL expression contains an unsupported use of CASE")
214+
215+// ErrParseUnsupportedAlias is an error that occurs if there is an unsupported
216+// use of Alias.
217+var ErrParseUnsupportedAlias = errors.New("The SQL expression contains an unsupported use of ALIAS")
218+
219+// ErrParseUnsupportedSyntax is an error that occurs if there is an
220+// UnsupportedSyntax in the SQL expression.
221+var ErrParseUnsupportedSyntax = errors.New("The SQL expression contains unsupported syntax")
222+
223+// ErrParseUnknownOperator is an error that occurs if there is an invalid
224+// operator present in the SQL expression.
225+var ErrParseUnknownOperator = errors.New("The SQL expression contains an invalid operator")
226+
227+// ErrParseInvalidPathComponent is an error that occurs if there is an invalid
228+// path component.
229+var ErrParseInvalidPathComponent = errors.New("The SQL expression contains an invalid path component")
230+
231+// ErrParseMissingIdentAfterAt is an error that occurs if the wrong symbol
232+// follows the "@" symbol in the SQL expression.
233+var ErrParseMissingIdentAfterAt = errors.New("Did not find the expected identifier after the @ symbol in the SQL expression")
234+
235+// ErrParseUnexpectedOperator is an error that occurs if the SQL expression
236+// contains an unexpected operator.
237+var ErrParseUnexpectedOperator = errors.New("The SQL expression contains an unexpected operator")
238+
239+// ErrParseUnexpectedTerm is an error that occurs if the SQL expression contains
240+// an unexpected term.
241+var ErrParseUnexpectedTerm = errors.New("The SQL expression contains an unexpected term")
242+
243+// ErrParseUnexpectedToken is an error that occurs if the SQL expression
244+// contains an unexpected token.
245+var ErrParseUnexpectedToken = errors.New("The SQL expression contains an unexpected token")
246+
247+// ErrParseUnexpectedKeyword is an error that occurs if the SQL expression
248+// contains an unexpected keyword.
249+var ErrParseUnexpectedKeyword = errors.New("The SQL expression contains an unexpected keyword")
250+
251+// ErrParseExpectedExpression is an error that occurs if the SQL expression is
252+// not found.
253+var ErrParseExpectedExpression = errors.New("Did not find the expected SQL expression")
254+
255+// ErrParseExpectedLeftParenAfterCast is an error that occurs if the left
256+// parenthesis is missing after a cast in the SQL expression.
257+var ErrParseExpectedLeftParenAfterCast = errors.New("Did not find the expected left parenthesis after CAST in the SQL expression")
258+
259+// ErrParseExpectedLeftParenValueConstructor is an error that occurs if the left
260+// parenthesis is not found in the SQL expression.
261+var ErrParseExpectedLeftParenValueConstructor = errors.New("Did not find expected the left parenthesis in the SQL expression")
262+
263+// ErrParseExpectedLeftParenBuiltinFunctionCall is an error that occurs if the
264+// left parenthesis is not found in the SQL expression function call.
265+var ErrParseExpectedLeftParenBuiltinFunctionCall = errors.New("Did not find the expected left parenthesis in the SQL expression")
266+
267+// ErrParseExpectedArgumentDelimiter is an error that occurs if the argument
268+// delimiter for the SQL expression is not provided.
269+var ErrParseExpectedArgumentDelimiter = errors.New("Did not find the expected argument delimiter in the SQL expression")
270+
271+// ErrParseCastArity is an error that occurs because the CAST has incorrect
272+// arity.
273+var ErrParseCastArity = errors.New("The SQL expression CAST has incorrect arity")
274+
275+// ErrParseInvalidTypeParam is an error that occurs because there is an invalid
276+// parameter value.
277+var ErrParseInvalidTypeParam = errors.New("The SQL expression contains an invalid parameter value")
278+
279+// ErrParseEmptySelect is an error that occurs because the SQL expression
280+// contains an empty Select
281+var ErrParseEmptySelect = errors.New("The SQL expression contains an empty SELECT")
282+
283+// ErrParseSelectMissingFrom is an error that occurs because there is a missing
284+// From after the Select List.
285+var ErrParseSelectMissingFrom = errors.New("The SQL expression contains a missing FROM after SELECT list")
286+
287+// ErrParseExpectedIdentForGroupName is an error that occurs because Group is
288+// not supported in the SQL expression.
289+var ErrParseExpectedIdentForGroupName = errors.New("GROUP is not supported in the SQL expression")
290+
291+// ErrParseExpectedIdentForAlias is an error that occurs if expected identifier
292+// for alias is not in the SQL expression.
293+var ErrParseExpectedIdentForAlias = errors.New("Did not find the expected identifier for the alias in the SQL expression")
294+
295+// ErrParseUnsupportedCallWithStar is an error that occurs if COUNT is used with
296+// an argument other than "*".
297+var ErrParseUnsupportedCallWithStar = errors.New("Only COUNT with (*) as a parameter is supported in the SQL expression")
298+
299+// ErrParseNonUnaryAgregateFunctionCall is an error that occurs if more than one
300+// argument is provided as an argument for aggregation functions.
301+var ErrParseNonUnaryAgregateFunctionCall = errors.New("Only one argument is supported for aggregate functions in the SQL expression")
302+
303+// ErrParseMalformedJoin is an error that occurs if a "join" operation is
304+// attempted in the SQL expression as this is not supported.
305+var ErrParseMalformedJoin = errors.New("JOIN is not supported in the SQL expression")
306+
307+// ErrParseExpectedIdentForAt is an error that occurs if after "AT" an Alias
308+// identifier is not provided.
309+var ErrParseExpectedIdentForAt = errors.New("Did not find the expected identifier for AT name in the SQL expression")
310+
311+// ErrParseAsteriskIsNotAloneInSelectList is an error that occurs if in addition
312+// to an asterix, more column names are provided as arguments in the SQL
313+// expression.
314+var ErrParseAsteriskIsNotAloneInSelectList = errors.New("Other expressions are not allowed in the SELECT list when '*' is used without dot notation in the SQL expression")
315+
316+// ErrParseCannotMixSqbAndWildcardInSelectList is an error that occurs if list
317+// indexing and an asterix are mixed in the SQL expression.
318+var ErrParseCannotMixSqbAndWildcardInSelectList = errors.New("Cannot mix [] and * in the same expression in a SELECT list in SQL expression")
319+
320+// ErrParseInvalidContextForWildcardInSelectList is an error that occurs if the
321+// asterix is used improperly within the SQL expression.
322+var ErrParseInvalidContextForWildcardInSelectList = errors.New("Invalid use of * in SELECT list in the SQL expression")
323+
324+// ErrEvaluatorBindingDoesNotExist is an error that occurs if a column name or
325+// path provided in the expression does not exist.
326+var ErrEvaluatorBindingDoesNotExist = errors.New("A column name or a path provided does not exist in the SQL expression")
327+
328+// ErrIncorrectSQLFunctionArgumentType is an error that occurs if the wrong
329+// argument is provided to a SQL function.
330+var ErrIncorrectSQLFunctionArgumentType = errors.New("Incorrect type of arguments in function call in the SQL expression")
331+
332+// ErrAmbiguousFieldName is an error that occurs if the column name which is not
333+// case sensitive, is not descriptive enough to retrieve a singular column.
334+var ErrAmbiguousFieldName = errors.New("Field name matches to multiple fields in the file. Check the SQL expression and the file, and try again")
335+
336+// ErrEvaluatorInvalidArguments is an error that occurs if there are not the
337+// correct number of arguments in a functional call to a SQL expression.
338+var ErrEvaluatorInvalidArguments = errors.New("Incorrect number of arguments in the function call in the SQL expression")
339+
340+// ErrValueParseFailure is an error that occurs if the Time Stamp is not parsed
341+// correctly in the SQL expression.
342+var ErrValueParseFailure = errors.New("Time stamp parse failure in the SQL expression")
343+
344+// ErrIntegerOverflow is an error that occurs if there is an IntegerOverflow or
345+// IntegerUnderFlow in the SQL expression.
346+var ErrIntegerOverflow = errors.New("Int overflow or underflow in the SQL expression")
347+
348+// ErrLikeInvalidInputs is an error that occurs if invalid inputs are provided
349+// to the argument LIKE Clause.
350+var ErrLikeInvalidInputs = errors.New("Invalid argument given to the LIKE clause in the SQL expression")
351+
352+// ErrCastFailed occurs if the attempt to convert data types in the cast is not
353+// done correctly.
354+var ErrCastFailed = errors.New("Attempt to convert from one data type to another using CAST failed in the SQL expression")
355+
356+// ErrInvalidCast is an error that occurs if the attempt to convert data types
357+// failed and was done in an improper fashion.
358+var ErrInvalidCast = errors.New("Attempt to convert from one data type to another using CAST failed in the SQL expression")
359+
360+// ErrEvaluatorInvalidTimestampFormatPattern is an error that occurs if the Time
361+// Stamp Format needs more additional fields to be filled.
362+var ErrEvaluatorInvalidTimestampFormatPattern = errors.New("Time stamp format pattern requires additional fields in the SQL expression")
363+
364+// ErrEvaluatorInvalidTimestampFormatPatternSymbolForParsing is an error that
365+// occurs if the format of the time stamp can not be parsed.
366+var ErrEvaluatorInvalidTimestampFormatPatternSymbolForParsing = errors.New("Time stamp format pattern contains a valid format symbol that cannot be applied to time stamp parsing in the SQL expression")
367+
368+// ErrEvaluatorTimestampFormatPatternDuplicateFields is an error that occurs if
369+// the time stamp format pattern contains multiple format specifications which
370+// can not be clearly resolved.
371+var ErrEvaluatorTimestampFormatPatternDuplicateFields = errors.New("Time stamp format pattern contains multiple format specifiers representing the time stamp field in the SQL expression")
372+
373+//ErrEvaluatorTimestampFormatPatternHourClockAmPmMismatch is an error that
374+//occurs if the time stamp format pattern contains a 12 hour day of format but
375+//does not have an AM/PM field.
376+var ErrEvaluatorTimestampFormatPatternHourClockAmPmMismatch = errors.New("Time stamp format pattern contains a 12-hour hour of day format symbol but doesn't also contain an AM/PM field, or it contains a 24-hour hour of day format specifier and contains an AM/PM field in the SQL expression")
377+
378+// ErrEvaluatorUnterminatedTimestampFormatPatternToken is an error that occurs
379+// if there is an unterminated token in the SQL expression for time stamp
380+// format.
381+var ErrEvaluatorUnterminatedTimestampFormatPatternToken = errors.New("Time stamp format pattern contains unterminated token in the SQL expression")
382+
383+// ErrEvaluatorInvalidTimestampFormatPatternToken is an error that occurs if
384+// there is an invalid token in the time stamp format within the SQL expression.
385+var ErrEvaluatorInvalidTimestampFormatPatternToken = errors.New("Time stamp format pattern contains an invalid token in the SQL expression")
386+
387+// ErrEvaluatorInvalidTimestampFormatPatternSymbol is an error that occurs if
388+// the time stamp format pattern has an invalid symbol within the SQL
389+// expression.
390+var ErrEvaluatorInvalidTimestampFormatPatternSymbol = errors.New("Time stamp format pattern contains an invalid symbol in the SQL expression")
391+
392+// S3 select API errors - TODO fix the errors.
393+var errorCodeResponse = map[error]string{
394+ ErrBusy: "Busy",
395+ ErrUnauthorizedAccess: "UnauthorizedAccess",
396+ ErrExpressionTooLong: "ExpressionTooLong",
397+ ErrIllegalSQLFunctionArgument: "IllegalSqlFunctionArgument",
398+ ErrInvalidColumnIndex: "InvalidColumnIndex",
399+ ErrInvalidKeyPath: "InvalidKeyPath",
400+ ErrColumnTooLong: "ColumnTooLong",
401+ ErrOverMaxColumn: "OverMaxColumn",
402+ ErrOverMaxRecordSize: "OverMaxRecordSize",
403+ ErrMissingHeaders: "MissingHeaders",
404+ ErrInvalidCompressionFormat: "InvalidCompressionFormat",
405+ ErrTruncatedInput: "TruncatedInput",
406+ ErrInvalidFileHeaderInfo: "InvalidFileHeaderInfo",
407+ ErrInvalidJSONType: "InvalidJsonType",
408+ ErrInvalidQuoteFields: "InvalidQuoteFields",
409+ ErrInvalidRequestParameter: "InvalidRequestParameter",
410+ ErrCSVParsingError: "CSVParsingError",
411+ ErrJSONParsingError: "JSONParsingError",
412+ ErrExternalEvalException: "ExternalEvalException",
413+ ErrInvalidDataType: "InvalidDataType",
414+ ErrUnrecognizedFormatException: "UnrecognizedFormatException",
415+ ErrInvalidTextEncoding: "InvalidTextEncoding",
416+ ErrInvalidTableAlias: "InvalidTableAlias",
417+ ErrMultipleDataSourcesUnsupported: "MultipleDataSourcesUnsupported",
418+ ErrMissingRequiredParameter: "MissingRequiredParameter",
419+ ErrObjectSerializationConflict: "ObjectSerializationConflict",
420+ ErrUnsupportedSQLOperation: "UnsupportedSqlOperation",
421+ ErrUnsupportedSQLStructure: "UnsupportedSqlStructure",
422+ ErrUnsupportedStorageClass: "UnsupportedStorageClass",
423+ ErrUnsupportedSyntax: "UnsupportedSyntax",
424+ ErrUnsupportedRangeHeader: "UnsupportedRangeHeader",
425+ ErrLexerInvalidChar: "LexerInvalidChar",
426+ ErrLexerInvalidOperator: "LexerInvalidOperator",
427+ ErrLexerInvalidLiteral: "LexerInvalidLiteral",
428+ ErrLexerInvalidIONLiteral: "LexerInvalidIONLiteral",
429+ ErrParseExpectedDatePart: "ParseExpectedDatePart",
430+ ErrParseExpectedKeyword: "ParseExpectedKeyword",
431+ ErrParseExpectedTokenType: "ParseExpectedTokenType",
432+ ErrParseExpected2TokenTypes: "ParseExpected2TokenTypes",
433+ ErrParseExpectedNumber: "ParseExpectedNumber",
434+ ErrParseExpectedRightParenBuiltinFunctionCall: "ParseExpectedRightParenBuiltinFunctionCall",
435+ ErrParseExpectedTypeName: "ParseExpectedTypeName",
436+ ErrParseExpectedWhenClause: "ParseExpectedWhenClause",
437+ ErrParseUnsupportedToken: "ParseUnsupportedToken",
438+ ErrParseUnsupportedLiteralsGroupBy: "ParseUnsupportedLiteralsGroupBy",
439+ ErrParseExpectedMember: "ParseExpectedMember",
440+ ErrParseUnsupportedSelect: "ParseUnsupportedSelect",
441+ ErrParseUnsupportedCase: "ParseUnsupportedCase:",
442+ ErrParseUnsupportedCaseClause: "ParseUnsupportedCaseClause",
443+ ErrParseUnsupportedAlias: "ParseUnsupportedAlias",
444+ ErrParseUnsupportedSyntax: "ParseUnsupportedSyntax",
445+ ErrParseUnknownOperator: "ParseUnknownOperator",
446+ ErrParseInvalidPathComponent: "ParseInvalidPathComponent",
447+ ErrParseMissingIdentAfterAt: "ParseMissingIdentAfterAt",
448+ ErrParseUnexpectedOperator: "ParseUnexpectedOperator",
449+ ErrParseUnexpectedTerm: "ParseUnexpectedTerm",
450+ ErrParseUnexpectedToken: "ParseUnexpectedToken",
451+ ErrParseUnexpectedKeyword: "ParseUnexpectedKeyword",
452+ ErrParseExpectedExpression: "ParseExpectedExpression",
453+ ErrParseExpectedLeftParenAfterCast: "ParseExpectedLeftParenAfterCast",
454+ ErrParseExpectedLeftParenValueConstructor: "ParseExpectedLeftParenValueConstructor",
455+ ErrParseExpectedLeftParenBuiltinFunctionCall: "ParseExpectedLeftParenBuiltinFunctionCall",
456+ ErrParseExpectedArgumentDelimiter: "ParseExpectedArgumentDelimiter",
457+ ErrParseCastArity: "ParseCastArity",
458+ ErrParseInvalidTypeParam: "ParseInvalidTypeParam",
459+ ErrParseEmptySelect: "ParseEmptySelect",
460+ ErrParseSelectMissingFrom: "ParseSelectMissingFrom",
461+ ErrParseExpectedIdentForGroupName: "ParseExpectedIdentForGroupName",
462+ ErrParseExpectedIdentForAlias: "ParseExpectedIdentForAlias",
463+ ErrParseUnsupportedCallWithStar: "ParseUnsupportedCallWithStar",
464+ ErrParseNonUnaryAgregateFunctionCall: "ParseNonUnaryAgregateFunctionCall",
465+ ErrParseMalformedJoin: "ParseMalformedJoin",
466+ ErrParseExpectedIdentForAt: "ParseExpectedIdentForAt",
467+ ErrParseAsteriskIsNotAloneInSelectList: "ParseAsteriskIsNotAloneInSelectList",
468+ ErrParseCannotMixSqbAndWildcardInSelectList: "ParseCannotMixSqbAndWildcardInSelectList",
469+ ErrParseInvalidContextForWildcardInSelectList: "ParseInvalidContextForWildcardInSelectList",
470+ ErrEvaluatorBindingDoesNotExist: "EvaluatorBindingDoesNotExist",
471+ ErrIncorrectSQLFunctionArgumentType: "IncorrectSqlFunctionArgumentType",
472+ ErrAmbiguousFieldName: "AmbiguousFieldName",
473+ ErrEvaluatorInvalidArguments: "EvaluatorInvalidArguments",
474+ ErrValueParseFailure: "ValueParseFailure",
475+ ErrIntegerOverflow: "IntegerOverflow",
476+ ErrLikeInvalidInputs: "LikeInvalidInputs",
477+ ErrCastFailed: "CastFailed",
478+ ErrInvalidCast: "Attempt to convert from one data type to another using CAST failed in the SQL expression.",
479+ ErrEvaluatorInvalidTimestampFormatPattern: "EvaluatorInvalidTimestampFormatPattern",
480+ ErrEvaluatorInvalidTimestampFormatPatternSymbolForParsing: "EvaluatorInvalidTimestampFormatPatternSymbolForParsing",
481+ ErrEvaluatorTimestampFormatPatternDuplicateFields: "EvaluatorTimestampFormatPatternDuplicateFields",
482+ ErrEvaluatorTimestampFormatPatternHourClockAmPmMismatch: "EvaluatorTimestampFormatPatternHourClockAmPmMismatch",
483+ ErrEvaluatorUnterminatedTimestampFormatPatternToken: "EvaluatorUnterminatedTimestampFormatPatternToken",
484+ ErrEvaluatorInvalidTimestampFormatPatternToken: "EvaluatorInvalidTimestampFormatPatternToken",
485+ ErrEvaluatorInvalidTimestampFormatPatternSymbol: "EvaluatorInvalidTimestampFormatPatternSymbol",
486+}
pkg/s3select/funcEval.go+231−0
@@ -0,0 +1,231 @@
1+/*
2+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
3+ *
4+ * Licensed under the Apache License, Version 2.0 (the "License");
5+ * you may not use this file except in compliance with the License.
6+ * You may obtain a copy of the License at
7+ *
8+ * http://www.apache.org/licenses/LICENSE-2.0
9+ *
10+ * Unless required by applicable law or agreed to in writing, software
11+ * distributed under the License is distributed on an "AS IS" BASIS,
12+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+ * See the License for the specific language governing permissions and
14+ * limitations under the License.
15+ */
16+
17+package s3select
18+
19+import (
20+ "strings"
21+
22+ "github.com/xwb1989/sqlparser"
23+)
24+
25+// stringOps is a function which handles the case in a clause if there is a need
26+// to perform a string function
27+func stringOps(myFunc *sqlparser.FuncExpr, record []string, myReturnVal string, columnsMap map[string]int) string {
28+ var value string
29+ funcName := myFunc.Name.CompliantName()
30+ switch tempArg := myFunc.Exprs[0].(type) {
31+ case *sqlparser.AliasedExpr:
32+ switch col := tempArg.Expr.(type) {
33+ case *sqlparser.FuncExpr:
34+ // myReturnVal is actually the tail recursive value being used in the eval func.
35+ return applyStrFunc(myReturnVal, funcName)
36+ case *sqlparser.ColName:
37+ value = applyStrFunc(record[columnsMap[col.Name.CompliantName()]], funcName)
38+ case *sqlparser.SQLVal:
39+ value = applyStrFunc(string(col.Val), funcName)
40+ }
41+ }
42+ return value
43+}
44+
45+// coalOps is a function which decomposes a COALESCE func expr into its struct.
46+func coalOps(myFunc *sqlparser.FuncExpr, record []string, myReturnVal string, columnsMap map[string]int) string {
47+ myArgs := make([]string, len(myFunc.Exprs))
48+
49+ for i := 0; i < len(myFunc.Exprs); i++ {
50+ switch tempArg := myFunc.Exprs[i].(type) {
51+ case *sqlparser.AliasedExpr:
52+ switch col := tempArg.Expr.(type) {
53+ case *sqlparser.FuncExpr:
54+ // myReturnVal is actually the tail recursive value being used in the eval func.
55+ return myReturnVal
56+ case *sqlparser.ColName:
57+ myArgs[i] = record[columnsMap[col.Name.CompliantName()]]
58+ case *sqlparser.SQLVal:
59+ myArgs[i] = string(col.Val)
60+ }
61+ }
62+ }
63+ return processCoalNoIndex(myArgs)
64+}
65+
66+// nullOps is a function which decomposes a NullIf func expr into its struct.
67+func nullOps(myFunc *sqlparser.FuncExpr, record []string, myReturnVal string, columnsMap map[string]int) string {
68+ myArgs := make([]string, 2)
69+
70+ for i := 0; i < len(myFunc.Exprs); i++ {
71+ switch tempArg := myFunc.Exprs[i].(type) {
72+ case *sqlparser.AliasedExpr:
73+ switch col := tempArg.Expr.(type) {
74+ case *sqlparser.FuncExpr:
75+ return myReturnVal
76+ case *sqlparser.ColName:
77+ myArgs[i] = record[columnsMap[col.Name.CompliantName()]]
78+ case *sqlparser.SQLVal:
79+ myArgs[i] = string(col.Val)
80+ }
81+ }
82+ }
83+ return processNullIf(myArgs)
84+}
85+
86+// isValidString is a function that ensures the current index is one with a
87+// StrFunc
88+func isValidFunc(myList []int, index int) bool {
89+ if myList == nil {
90+ return false
91+ }
92+ for i := 0; i < len(myList); i++ {
93+ if myList[i] == index {
94+ return true
95+ }
96+ }
97+ return false
98+}
99+
100+// processNullIf is a function that evaluates a given NULLIF clause.
101+func processNullIf(nullStore []string) string {
102+ nullValOne := nullStore[0]
103+ nullValTwo := nullStore[1]
104+ if nullValOne == nullValTwo {
105+ return ""
106+ }
107+ return nullValOne
108+}
109+
110+// processCoalNoIndex is a function which evaluates a given COALESCE clause.
111+func processCoalNoIndex(coalStore []string) string {
112+ for i := 0; i < len(coalStore); i++ {
113+ if coalStore[i] != "null" && coalStore[i] != "missing" && coalStore[i] != "" {
114+ return coalStore[i]
115+ }
116+ }
117+ return "null"
118+}
119+
120+// evaluateFuncExpr is a function that allows for tail recursive evaluation of
121+// nested function expressions.
122+func evaluateFuncExpr(myVal *sqlparser.FuncExpr, myReturnVal string, myRecord []string, columnsMap map[string]int) string {
123+ if myVal == nil {
124+ return myReturnVal
125+ }
126+ // retrieve all the relevant arguments of the function
127+ var mySubFunc []*sqlparser.FuncExpr
128+ mySubFunc = make([]*sqlparser.FuncExpr, len(myVal.Exprs))
129+ for i := 0; i < len(myVal.Exprs); i++ {
130+ switch col := myVal.Exprs[i].(type) {
131+ case *sqlparser.AliasedExpr:
132+ switch temp := col.Expr.(type) {
133+ case *sqlparser.FuncExpr:
134+ mySubFunc[i] = temp
135+ }
136+ }
137+ }
138+ // Need to do tree recursion so as to explore all possible directions of the
139+ // nested function recursion
140+ for i := 0; i < len(mySubFunc); i++ {
141+ if supportedString(myVal.Name.CompliantName()) {
142+ if mySubFunc != nil {
143+ return stringOps(myVal, myRecord, evaluateFuncExpr(mySubFunc[i], myReturnVal, myRecord, columnsMap), columnsMap)
144+ }
145+ return stringOps(myVal, myRecord, myReturnVal, columnsMap)
146+ } else if strings.ToUpper(myVal.Name.CompliantName()) == "NULLIF" {
147+ if mySubFunc != nil {
148+ return nullOps(myVal, myRecord, evaluateFuncExpr(mySubFunc[i], myReturnVal, myRecord, columnsMap), columnsMap)
149+ }
150+ return nullOps(myVal, myRecord, myReturnVal, columnsMap)
151+ } else if strings.ToUpper(myVal.Name.CompliantName()) == "COALESCE" {
152+ if mySubFunc != nil {
153+ return coalOps(myVal, myRecord, evaluateFuncExpr(mySubFunc[i], myReturnVal, myRecord, columnsMap), columnsMap)
154+ }
155+ return coalOps(myVal, myRecord, myReturnVal, columnsMap)
156+ }
157+ }
158+ return ""
159+}
160+
161+// evaluateFuncErr is a function that flags errors in nested functions.
162+func (reader *Input) evaluateFuncErr(myVal *sqlparser.FuncExpr) error {
163+ if myVal == nil {
164+ return nil
165+ }
166+ if !supportedFunc(myVal.Name.CompliantName()) {
167+ return ErrUnsupportedSQLOperation
168+ }
169+ for i := 0; i < len(myVal.Exprs); i++ {
170+ switch tempArg := myVal.Exprs[i].(type) {
171+ case *sqlparser.StarExpr:
172+ return ErrParseUnsupportedCallWithStar
173+ case *sqlparser.AliasedExpr:
174+ switch col := tempArg.Expr.(type) {
175+ case *sqlparser.FuncExpr:
176+ if err := reader.evaluateFuncErr(col); err != nil {
177+ return err
178+ }
179+ case *sqlparser.ColName:
180+ if err := reader.colNameErrs([]string{col.Name.CompliantName()}); err != nil {
181+ return err
182+ }
183+ }
184+ }
185+ }
186+ return nil
187+}
188+
189+// evaluateIsExpr is a function for evaluating expressions of the form "column
190+// is ...."
191+func evaluateIsExpr(myFunc *sqlparser.IsExpr, row []string, columnNames map[string]int, alias string) (bool, error) {
192+ operator := myFunc.Operator
193+ var colName string
194+ var myVal string
195+ switch myIs := myFunc.Expr.(type) {
196+ // case for literal val
197+ case *sqlparser.SQLVal:
198+ myVal = string(myIs.Val)
199+ // case for nested func val
200+ case *sqlparser.FuncExpr:
201+ myVal = evaluateFuncExpr(myIs, "", row, columnNames)
202+ // case for col val
203+ case *sqlparser.ColName:
204+ colName = cleanCol(myIs.Name.CompliantName(), alias)
205+ }
206+ // case if it is a col val
207+ if colName != "" {
208+ myVal = row[columnNames[colName]]
209+ }
210+ // case to evaluate is null
211+ if strings.ToLower(operator) == "is null" {
212+ return myVal == "", nil
213+ }
214+ // case to evaluate is not null
215+ if strings.ToLower(operator) == "is not null" {
216+ return myVal != "", nil
217+ }
218+ return false, ErrUnsupportedSQLOperation
219+}
220+
221+// supportedString is a function that checks whether the function is a supported
222+// string one
223+func supportedString(strFunc string) bool {
224+ return stringInSlice(strings.ToUpper(strFunc), []string{"TRIM", "SUBSTRING", "CHAR_LENGTH", "CHARACTER_LENGTH", "LOWER", "UPPER"})
225+}
226+
227+// supportedFunc is a function that checks whether the function is a supported
228+// S3 one.
229+func supportedFunc(strFunc string) bool {
230+ return stringInSlice(strings.ToUpper(strFunc), []string{"TRIM", "SUBSTRING", "CHAR_LENGTH", "CHARACTER_LENGTH", "LOWER", "UPPER", "COALESCE", "NULLIF"})
231+}
pkg/s3select/helpers.go+754−0
@@ -0,0 +1,754 @@
1+/*
2+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
3+ *
4+ * Licensed under the Apache License, Version 2.0 (the "License");
5+ * you may not use this file except in compliance with the License.
6+ * You may obtain a copy of the License at
7+ *
8+ * http://www.apache.org/licenses/LICENSE-2.0
9+ *
10+ * Unless required by applicable law or agreed to in writing, software
11+ * distributed under the License is distributed on an "AS IS" BASIS,
12+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+ * See the License for the specific language governing permissions and
14+ * limitations under the License.
15+ */
16+
17+package s3select
18+
19+import (
20+ "fmt"
21+ "reflect"
22+ "strconv"
23+ "strings"
24+
25+ "github.com/xwb1989/sqlparser"
26+)
27+
28+// This function processes size so that we can calculate bytes BytesProcessed.
29+func processSize(myrecord []string) int64 {
30+ if len(myrecord) > 0 {
31+ var size int64
32+ size = int64(len(myrecord)-1) + 1
33+ for i := range myrecord {
34+ size += int64(len(myrecord[i]))
35+ }
36+
37+ return size
38+ }
39+ return 0
40+}
41+
42+// This function finds whether a string is in a list
43+func stringInSlice(x string, list []string) bool {
44+ for _, y := range list {
45+ if x == y {
46+ return true
47+ }
48+ }
49+ return false
50+}
51+
52+// This function returns the index of a string in a list
53+func stringIndex(a string, list []string) int {
54+ for i := range list {
55+ if list[i] == a {
56+ return i
57+ }
58+ }
59+ return -1
60+}
61+
62+// Returns a true or false, whether a string can be represented as an int.
63+func representsInt(s string) bool {
64+ if _, err := strconv.Atoi(s); err == nil {
65+ return true
66+ }
67+ return false
68+}
69+
70+// The function below processes the where clause into an acutal boolean given a
71+// row
72+func matchesMyWhereClause(row []string, columnNames map[string]int, alias string, whereClause interface{}) (bool, error) {
73+ // This particular logic deals with the details of casting, e.g if we have to
74+ // cast a column of string numbers into int's for comparison.
75+ var conversionColumn string
76+ var operator string
77+ var operand interface{}
78+ if fmt.Sprintf("%v", whereClause) == "false" {
79+ return false, nil
80+ }
81+ switch expr := whereClause.(type) {
82+ case *sqlparser.IsExpr:
83+ return evaluateIsExpr(expr, row, columnNames, alias)
84+ case *sqlparser.RangeCond:
85+ operator = expr.Operator
86+ if operator != "between" && operator != "not between" {
87+ return false, ErrUnsupportedSQLOperation
88+ }
89+ if operator == "not between" {
90+ myResult, err := evaluateBetween(expr, alias, row, columnNames)
91+ if err != nil {
92+ return false, err
93+ }
94+ return !myResult, nil
95+ }
96+ myResult, err := evaluateBetween(expr, alias, row, columnNames)
97+ if err != nil {
98+ return false, err
99+ }
100+ return myResult, nil
101+ case *sqlparser.ComparisonExpr:
102+ operator = expr.Operator
103+ switch right := expr.Right.(type) {
104+ case *sqlparser.FuncExpr:
105+ operand = evaluateFuncExpr(right, "", row, columnNames)
106+ case *sqlparser.SQLVal:
107+ var err error
108+ operand, err = evaluateParserType(right)
109+ if err != nil {
110+ return false, err
111+ }
112+ }
113+ var myVal string
114+ myVal = ""
115+ switch left := expr.Left.(type) {
116+ case *sqlparser.FuncExpr:
117+ myVal = evaluateFuncExpr(left, "", row, columnNames)
118+ conversionColumn = ""
119+ case *sqlparser.ColName:
120+ conversionColumn = cleanCol(left.Name.CompliantName(), alias)
121+ }
122+ if representsInt(conversionColumn) {
123+ intCol, err := strconv.Atoi(conversionColumn)
124+ if err != nil {
125+ return false, err
126+ }
127+ // Subtract 1 out because the index starts at 1 for Amazon instead of 0.
128+ return evaluateOperator(row[intCol-1], operator, operand)
129+ }
130+ if myVal != "" {
131+ return evaluateOperator(myVal, operator, operand)
132+ }
133+ return evaluateOperator(row[columnNames[conversionColumn]], operator, operand)
134+ case *sqlparser.AndExpr:
135+ var leftVal bool
136+ var rightVal bool
137+ switch left := expr.Left.(type) {
138+ case *sqlparser.ComparisonExpr:
139+ temp, err := matchesMyWhereClause(row, columnNames, alias, left)
140+ if err != nil {
141+ return false, err
142+ }
143+ leftVal = temp
144+ }
145+ switch right := expr.Right.(type) {
146+ case *sqlparser.ComparisonExpr:
147+ temp, err := matchesMyWhereClause(row, columnNames, alias, right)
148+ if err != nil {
149+ return false, err
150+ }
151+ rightVal = temp
152+ }
153+ return (rightVal && leftVal), nil
154+ case *sqlparser.OrExpr:
155+ var leftVal bool
156+ var rightVal bool
157+ switch left := expr.Left.(type) {
158+ case *sqlparser.ComparisonExpr:
159+ leftVal, _ = matchesMyWhereClause(row, columnNames, alias, left)
160+
161+ }
162+ switch right := expr.Right.(type) {
163+ case *sqlparser.ComparisonExpr:
164+ rightVal, _ = matchesMyWhereClause(row, columnNames, alias, right)
165+ }
166+ return (rightVal || leftVal), nil
167+
168+ }
169+ return true, nil
170+}
171+func applyStrFunc(rawArg string, funcName string) string {
172+ switch strings.ToUpper(funcName) {
173+ case "TRIM":
174+ // parser has an issue which does not allow it to support Trim with other
175+ // arguments
176+ return strings.Trim(rawArg, " ")
177+ case "SUBSTRING":
178+ // TODO parser has an issue which does not support substring
179+ return rawArg
180+ case "CHAR_LENGTH":
181+ return strconv.Itoa(len(rawArg))
182+ case "CHARACTER_LENGTH":
183+ return strconv.Itoa(len(rawArg))
184+ case "LOWER":
185+ return strings.ToLower(rawArg)
186+ case "UPPER":
187+ return strings.ToUpper(rawArg)
188+ }
189+ return rawArg
190+
191+}
192+
193+// This is a really important function it actually evaluates the boolean
194+// statement and therefore actually returns a bool, it functions as the lowest
195+// level of the state machine.
196+func evaluateOperator(myTblVal string, operator string, operand interface{}) (bool, error) {
197+ if err := checkValidOperator(operator); err != nil {
198+ return false, err
199+ }
200+ myRecordVal := checkStringType(myTblVal)
201+ myVal := reflect.ValueOf(myRecordVal)
202+ myOp := reflect.ValueOf(operand)
203+
204+ switch {
205+ case myVal.Kind() == reflect.String && myOp.Kind() == reflect.String:
206+ return stringEval(myVal.String(), operator, myOp.String())
207+ case myVal.Kind() == reflect.Float64 && myOp.Kind() == reflect.Float64:
208+ return floatEval(myVal.Float(), operator, myOp.Float())
209+ case myVal.Kind() == reflect.Int && myOp.Kind() == reflect.Int:
210+ return intEval(myVal.Int(), operator, myOp.Int())
211+ case myVal.Kind() == reflect.Int && myOp.Kind() == reflect.String:
212+ stringVs := strconv.Itoa(int(myVal.Int()))
213+ return stringEval(stringVs, operator, myOp.String())
214+ case myVal.Kind() == reflect.Float64 && myOp.Kind() == reflect.String:
215+ stringVs := strconv.FormatFloat(myVal.Float(), 'f', 6, 64)
216+ return stringEval(stringVs, operator, myOp.String())
217+ case myVal.Kind() != myOp.Kind():
218+ return false, nil
219+ }
220+ return false, ErrUnsupportedSyntax
221+}
222+
223+// checkValidOperator ensures that the current operator is supported
224+func checkValidOperator(operator string) error {
225+ listOfOps := []string{">", "<", "=", "<=", ">=", "!=", "like"}
226+ for i := range listOfOps {
227+ if operator == listOfOps[i] {
228+ return nil
229+ }
230+ }
231+ return ErrParseUnknownOperator
232+}
233+
234+// checkStringType converts the value from the csv to the appropriate one.
235+func checkStringType(myTblVal string) interface{} {
236+ myInt, isInt := strconv.Atoi(myTblVal)
237+ myFloat, isFloat := strconv.ParseFloat(myTblVal, 64)
238+ if isInt == nil {
239+ return myInt
240+ } else if isFloat == nil {
241+ return myFloat
242+ } else {
243+ return myTblVal
244+ }
245+}
246+
247+// stringEval is for evaluating the state of string comparison.
248+func stringEval(myRecordVal string, operator string, myOperand string) (bool, error) {
249+ switch operator {
250+ case ">":
251+ return myRecordVal > myOperand, nil
252+ case "<":
253+ return myRecordVal < myOperand, nil
254+ case "=":
255+ return myRecordVal == myOperand, nil
256+ case "<=":
257+ return myRecordVal <= myOperand, nil
258+ case ">=":
259+ return myRecordVal >= myOperand, nil
260+ case "!=":
261+ return myRecordVal != myOperand, nil
262+ case "like":
263+ return likeConvert(myOperand, myRecordVal)
264+ }
265+ return false, ErrUnsupportedSyntax
266+}
267+
268+// intEval is for evaluating integer comparisons.
269+func intEval(myRecordVal int64, operator string, myOperand int64) (bool, error) {
270+
271+ switch operator {
272+ case ">":
273+ return myRecordVal > myOperand, nil
274+ case "<":
275+ return myRecordVal < myOperand, nil
276+ case "=":
277+ return myRecordVal == myOperand, nil
278+ case "<=":
279+ return myRecordVal <= myOperand, nil
280+ case ">=":
281+ return myRecordVal >= myOperand, nil
282+ case "!=":
283+ return myRecordVal != myOperand, nil
284+ }
285+ return false, ErrUnsupportedSyntax
286+}
287+
288+// floatEval is for evaluating the comparison of floats.
289+func floatEval(myRecordVal float64, operator string, myOperand float64) (bool, error) {
290+ // Basically need some logic thats like, if the types dont match check for a cast
291+ switch operator {
292+ case ">":
293+ return myRecordVal > myOperand, nil
294+ case "<":
295+ return myRecordVal < myOperand, nil
296+ case "=":
297+ return myRecordVal == myOperand, nil
298+ case "<=":
299+ return myRecordVal <= myOperand, nil
300+ case ">=":
301+ return myRecordVal >= myOperand, nil
302+ case "!=":
303+ return myRecordVal != myOperand, nil
304+ }
305+ return false, ErrUnsupportedSyntax
306+}
307+
308+// prefixMatch allows for matching a prefix only like query e.g a%
309+func prefixMatch(pattern string, record string) bool {
310+ for i := 0; i < len(pattern)-1; i++ {
311+ if pattern[i] != record[i] && pattern[i] != byte('_') {
312+ return false
313+ }
314+ }
315+ return true
316+}
317+
318+// suffixMatch allows for matching a suffix only like query e.g %an
319+func suffixMatch(pattern string, record string) bool {
320+ for i := len(pattern) - 1; i > 0; i-- {
321+ if pattern[i] != record[len(record)-(len(pattern)-i)] && pattern[i] != byte('_') {
322+ return false
323+ }
324+ }
325+ return true
326+}
327+
328+// This function is for evaluating select statements which are case sensitive
329+func likeConvert(pattern string, record string) (bool, error) {
330+ // If pattern is empty just return false
331+ if pattern == "" || record == "" {
332+ return false, nil
333+ }
334+ // for suffix match queries e.g %a
335+ if len(pattern) >= 2 && pattern[0] == byte('%') && strings.Count(pattern, "%") == 1 {
336+ return suffixMatch(pattern, record), nil
337+ }
338+ // for prefix match queries e.g a%
339+ if len(pattern) >= 2 && pattern[len(pattern)-1] == byte('%') && strings.Count(pattern, "%") == 1 {
340+ return prefixMatch(pattern, record), nil
341+ }
342+ charCount := 0
343+ currPos := 0
344+ // Loop through the pattern so that a boolean can be returned
345+ for i := 0; i < len(pattern); i++ {
346+ if pattern[i] == byte('_') {
347+ // if its an underscore it can be anything so shift current position for
348+ // pattern and string
349+ charCount++
350+ // if there have been more characters in the pattern than record, clearly
351+ // there should be a return
352+ if i != len(pattern)-1 {
353+ if pattern[i+1] != byte('%') && pattern[i+1] != byte('_') {
354+ if currPos != len(record)-1 && pattern[i+1] != record[currPos+1] {
355+ return false, nil
356+ }
357+ }
358+ }
359+ if charCount > len(record) {
360+ return false, nil
361+ }
362+ // if the pattern has been fully evaluated, then just return.
363+ if len(pattern) == i+1 {
364+ return true, nil
365+ }
366+ i++
367+ currPos++
368+ }
369+ if pattern[i] == byte('%') || pattern[i] == byte('*') {
370+ // if there is a wildcard then want to return true if its last and flag it.
371+ if currPos == len(record) {
372+ return false, nil
373+ }
374+ if i+1 == len(pattern) {
375+ return true, nil
376+ }
377+ } else {
378+ charCount++
379+ matched := false
380+ // iterate through the pattern and check if there is a match for the
381+ // character
382+ for currPos < len(record) {
383+ if record[currPos] == pattern[i] || pattern[i] == byte('_') {
384+ matched = true
385+ break
386+ }
387+ currPos++
388+ }
389+ currPos++
390+ // if the character did not match then return should occur.
391+ if !matched {
392+ return false, nil
393+ }
394+ }
395+ }
396+ if charCount > len(record) {
397+ return false, nil
398+ }
399+ if currPos < len(record) {
400+ return false, nil
401+ }
402+ return true, nil
403+}
404+
405+// TrimQuotes allows the following to occur select "name", we need to trim the
406+// quotes to reference our map of columnNames.
407+func trimQuotes(s string) string {
408+ if len(s) >= 2 {
409+ if c := s[len(s)-1]; s[0] == c && (c == '"') {
410+ return s[1 : len(s)-1]
411+ }
412+ }
413+ return s
414+}
415+
416+// cleanCol cleans a column name from the parser so that the name is returned to
417+// original.
418+func cleanCol(myCol string, alias string) string {
419+ if len(myCol) <= 0 {
420+ return myCol
421+ }
422+ if !strings.HasPrefix(myCol, alias) && myCol[0] == '_' {
423+ myCol = alias + myCol
424+ }
425+
426+ if strings.Contains(myCol, ".") {
427+ myCol = strings.Replace(myCol, alias+"._", "", len(myCol))
428+ }
429+ myCol = strings.Replace(myCol, alias+"_", "", len(myCol))
430+ return myCol
431+}
432+
433+// evaluateBetween is a function which evaluates a Between Clause.
434+func evaluateBetween(betweenExpr *sqlparser.RangeCond, alias string, record []string, columnNames map[string]int) (bool, error) {
435+ var colToVal interface{}
436+ var colFromVal interface{}
437+ var conversionColumn string
438+ var funcName string
439+ switch colTo := betweenExpr.To.(type) {
440+ case sqlparser.Expr:
441+ switch colToMyVal := colTo.(type) {
442+ case *sqlparser.FuncExpr:
443+ var temp string
444+ temp = stringOps(colToMyVal, record, "", columnNames)
445+ colToVal = []byte(temp)
446+ case *sqlparser.SQLVal:
447+ var err error
448+ colToVal, err = evaluateParserType(colToMyVal)
449+ if err != nil {
450+ return false, err
451+ }
452+ }
453+ }
454+ switch colFrom := betweenExpr.From.(type) {
455+ case sqlparser.Expr:
456+ switch colFromMyVal := colFrom.(type) {
457+ case *sqlparser.FuncExpr:
458+ colFromVal = stringOps(colFromMyVal, record, "", columnNames)
459+ case *sqlparser.SQLVal:
460+ var err error
461+ colFromVal, err = evaluateParserType(colFromMyVal)
462+ if err != nil {
463+ return false, err
464+ }
465+ }
466+ }
467+ var myFuncVal string
468+ myFuncVal = ""
469+ switch left := betweenExpr.Left.(type) {
470+ case *sqlparser.FuncExpr:
471+ myFuncVal = evaluateFuncExpr(left, "", record, columnNames)
472+ conversionColumn = ""
473+ case *sqlparser.ColName:
474+ conversionColumn = cleanCol(left.Name.CompliantName(), alias)
475+ }
476+
477+ toGreater, err := evaluateOperator(fmt.Sprintf("%v", colToVal), ">", colFromVal)
478+ if err != nil {
479+ return false, err
480+ }
481+ if toGreater {
482+ return evalBetweenGreater(conversionColumn, record, funcName, columnNames, colFromVal, colToVal, myFuncVal)
483+ }
484+ return evalBetweenLess(conversionColumn, record, funcName, columnNames, colFromVal, colToVal, myFuncVal)
485+}
486+
487+// evalBetweenLess is a function which evaluates the between given that the
488+// FROM is > than the TO.
489+func evalBetweenLess(conversionColumn string, record []string, funcName string, columnNames map[string]int, colFromVal interface{}, colToVal interface{}, myCoalVal string) (bool, error) {
490+ if representsInt(conversionColumn) {
491+ myIndex, _ := strconv.Atoi(conversionColumn)
492+ // Subtract 1 out because the index starts at 1 for Amazon instead of 0.
493+ myVal, err := evaluateOperator(record[myIndex-1], "<=", colFromVal)
494+ if err != nil {
495+ return false, err
496+ }
497+ var myOtherVal bool
498+ myOtherVal, err = evaluateOperator(fmt.Sprintf("%v", colToVal), "<=", checkStringType(record[myIndex-1]))
499+ if err != nil {
500+ return false, err
501+ }
502+ return (myVal && myOtherVal), nil
503+ }
504+ if myCoalVal != "" {
505+ myVal, err := evaluateOperator(myCoalVal, "<=", colFromVal)
506+ if err != nil {
507+ return false, err
508+ }
509+ var myOtherVal bool
510+ myOtherVal, err = evaluateOperator(fmt.Sprintf("%v", colToVal), "<=", checkStringType(myCoalVal))
511+ if err != nil {
512+ return false, err
513+ }
514+ return (myVal && myOtherVal), nil
515+ }
516+ myVal, err := evaluateOperator(record[columnNames[conversionColumn]], "<=", colFromVal)
517+ if err != nil {
518+ return false, err
519+ }
520+ var myOtherVal bool
521+ myOtherVal, err = evaluateOperator(fmt.Sprintf("%v", colToVal), "<=", checkStringType(record[columnNames[conversionColumn]]))
522+ if err != nil {
523+ return false, err
524+ }
525+ return (myVal && myOtherVal), nil
526+}
527+
528+// evalBetweenGreater is a function which evaluates the between given that the
529+// TO is > than the FROM.
530+func evalBetweenGreater(conversionColumn string, record []string, funcName string, columnNames map[string]int, colFromVal interface{}, colToVal interface{}, myCoalVal string) (bool, error) {
531+ if representsInt(conversionColumn) {
532+ myIndex, _ := strconv.Atoi(conversionColumn)
533+ myVal, err := evaluateOperator(record[myIndex-1], ">=", colFromVal)
534+ if err != nil {
535+ return false, err
536+ }
537+ var myOtherVal bool
538+ myOtherVal, err = evaluateOperator(fmt.Sprintf("%v", colToVal), ">=", checkStringType(record[myIndex-1]))
539+ if err != nil {
540+ return false, err
541+ }
542+ return (myVal && myOtherVal), nil
543+ }
544+ if myCoalVal != "" {
545+ myVal, err := evaluateOperator(myCoalVal, ">=", colFromVal)
546+ if err != nil {
547+ return false, err
548+ }
549+ var myOtherVal bool
550+ myOtherVal, err = evaluateOperator(fmt.Sprintf("%v", colToVal), ">=", checkStringType(myCoalVal))
551+ if err != nil {
552+ return false, err
553+ }
554+ return (myVal && myOtherVal), nil
555+ }
556+ myVal, err := evaluateOperator(record[columnNames[conversionColumn]], ">=", colFromVal)
557+ if err != nil {
558+ return false, err
559+ }
560+ var myOtherVal bool
561+ myOtherVal, err = evaluateOperator(fmt.Sprintf("%v", colToVal), ">=", checkStringType(record[columnNames[conversionColumn]]))
562+ if err != nil {
563+ return false, err
564+ }
565+ return (myVal && myOtherVal), nil
566+}
567+
568+// whereClauseNameErrs is a function which returns an error if there is a column
569+// in the where clause which does not exist.
570+func (reader *Input) whereClauseNameErrs(whereClause interface{}, alias string) error {
571+ var conversionColumn string
572+ switch expr := whereClause.(type) {
573+ // case for checking errors within a clause of the form "col_name is ..."
574+ case *sqlparser.IsExpr:
575+ switch myCol := expr.Expr.(type) {
576+ case *sqlparser.FuncExpr:
577+ if err := reader.evaluateFuncErr(myCol); err != nil {
578+ return err
579+ }
580+ case *sqlparser.ColName:
581+ conversionColumn = cleanCol(myCol.Name.CompliantName(), alias)
582+ }
583+ case *sqlparser.RangeCond:
584+ switch left := expr.Left.(type) {
585+ case *sqlparser.FuncExpr:
586+ if err := reader.evaluateFuncErr(left); err != nil {
587+ return err
588+ }
589+ case *sqlparser.ColName:
590+ conversionColumn = cleanCol(left.Name.CompliantName(), alias)
591+ }
592+ case *sqlparser.ComparisonExpr:
593+ switch left := expr.Left.(type) {
594+ case *sqlparser.FuncExpr:
595+ if err := reader.evaluateFuncErr(left); err != nil {
596+ return err
597+ }
598+ case *sqlparser.ColName:
599+ conversionColumn = cleanCol(left.Name.CompliantName(), alias)
600+ }
601+ case *sqlparser.AndExpr:
602+ switch left := expr.Left.(type) {
603+ case *sqlparser.ComparisonExpr:
604+ return reader.whereClauseNameErrs(left, alias)
605+ }
606+ switch right := expr.Right.(type) {
607+ case *sqlparser.ComparisonExpr:
608+ return reader.whereClauseNameErrs(right, alias)
609+ }
610+ case *sqlparser.OrExpr:
611+ switch left := expr.Left.(type) {
612+ case *sqlparser.ComparisonExpr:
613+ return reader.whereClauseNameErrs(left, alias)
614+ }
615+ switch right := expr.Right.(type) {
616+ case *sqlparser.ComparisonExpr:
617+ return reader.whereClauseNameErrs(right, alias)
618+ }
619+ }
620+ if conversionColumn != "" {
621+ return reader.colNameErrs([]string{conversionColumn})
622+ }
623+ return nil
624+}
625+
626+// qualityCheck ensures the row has enough separators.
627+func qualityCheck(row string, amountOfSep int, sep string) string {
628+ for i := 0; i < amountOfSep; i++ {
629+ row = row + sep
630+ }
631+ return row
632+}
633+
634+// writeRow helps to write the row regardless of how many entries.
635+func writeRow(myRow string, myEntry string, delimiter string, numOfReqCols int) string {
636+ if myEntry == "" && len(myRow) == 0 && numOfReqCols == 1 {
637+ return myEntry
638+ }
639+ if myEntry == "" && len(myRow) == 0 {
640+ return myEntry + delimiter
641+ }
642+ if len(myRow) == 1 && myRow[0] == ',' {
643+ return myRow + myEntry
644+ }
645+ if len(myRow) == 0 {
646+ return myEntry
647+ }
648+ return myRow + delimiter + myEntry
649+}
650+
651+// colNameErrs is a function which makes sure that the headers are requested are
652+// present in the file otherwise it throws an error.
653+func (reader *Input) colNameErrs(columnNames []string) error {
654+ for i := 0; i < len(columnNames); i++ {
655+ if columnNames[i] == "" {
656+ continue
657+ }
658+ if !representsInt(columnNames[i]) && !reader.options.HeaderOpt {
659+ return ErrInvalidColumnIndex
660+ }
661+ if representsInt(columnNames[i]) {
662+ tempInt, _ := strconv.Atoi(columnNames[i])
663+ if tempInt > len(reader.Header()) || tempInt == 0 {
664+ return ErrInvalidColumnIndex
665+ }
666+ } else {
667+ if reader.options.HeaderOpt && !stringInSlice(columnNames[i], reader.Header()) {
668+ return ErrMissingHeaders
669+ }
670+ }
671+ }
672+ return nil
673+}
674+
675+// aggFuncToStr converts an array of floats into a properly formatted string.
676+func (reader *Input) aggFuncToStr(myAggVals []float64) string {
677+ myRow := strconv.FormatFloat(myAggVals[0], 'f', 6, 64)
678+ for i := 1; i < len(myAggVals); i++ {
679+ aggregateval := strconv.FormatFloat(myAggVals[i], 'f', 6, 64)
680+ myRow = myRow + reader.options.OutputFieldDelimiter + aggregateval
681+ }
682+ return myRow
683+}
684+
685+// checkForDuplicates ensures we do not have an ambigious column name.
686+func checkForDuplicates(columns []string, columnsMap map[string]int, hasDuplicates map[string]bool, lowercaseColumnsMap map[string]int) error {
687+ for i := 0; i < len(columns); i++ {
688+ columns[i] = strings.Replace(columns[i], " ", "_", len(columns[i]))
689+ if _, exist := columnsMap[columns[i]]; exist {
690+ return ErrAmbiguousFieldName
691+ }
692+ columnsMap[columns[i]] = i
693+ // This checks that if a key has already been put into the map, that we're
694+ // setting its appropriate value in has duplicates to be true.
695+ if _, exist := lowercaseColumnsMap[strings.ToLower(columns[i])]; exist {
696+ hasDuplicates[strings.ToLower(columns[i])] = true
697+ } else {
698+ lowercaseColumnsMap[strings.ToLower(columns[i])] = i
699+ }
700+ }
701+ return nil
702+}
703+
704+// evaluateParserType is a function that takes a SQL value and returns it as an
705+// interface converted into the appropriate value.
706+func evaluateParserType(col *sqlparser.SQLVal) (interface{}, error) {
707+ colDataType := col.Type
708+ var val interface{}
709+ switch colDataType {
710+ case 0:
711+ val = string(col.Val)
712+ case 1:
713+ intVersion, isInt := strconv.Atoi(string(col.Val))
714+ if isInt != nil {
715+ return nil, ErrIntegerOverflow
716+ }
717+ val = intVersion
718+ case 2:
719+ floatVersion, isFloat := strconv.ParseFloat(string(col.Val), 64)
720+ if isFloat != nil {
721+ return nil, ErrIntegerOverflow
722+ }
723+ val = floatVersion
724+ }
725+ return val, nil
726+}
727+
728+// parseErrs is the function which handles all the errors that could occur
729+// through use of function arguments such as column names in NULLIF
730+func (reader *Input) parseErrs(columnNames []string, whereClause interface{}, alias string, myFuncs *SelectFuncs) error {
731+ // Below code cleans up column names.
732+ reader.processColumnNames(columnNames, alias)
733+ if columnNames[0] != "*" {
734+ if err := reader.colNameErrs(columnNames); err != nil {
735+ return err
736+ }
737+ }
738+ // Below code ensures the whereClause has no errors.
739+ if whereClause != nil {
740+ tempClause := whereClause
741+ if err := reader.whereClauseNameErrs(tempClause, alias); err != nil {
742+ return err
743+ }
744+ }
745+ for i := 0; i < len(myFuncs.funcExpr); i++ {
746+ if myFuncs.funcExpr[i] == nil {
747+ continue
748+ }
749+ if err := reader.evaluateFuncErr(myFuncs.funcExpr[i]); err != nil {
750+ return err
751+ }
752+ }
753+ return nil
754+}
pkg/s3select/input.go+381−0
@@ -0,0 +1,381 @@
1+/*
2+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
3+ *
4+ * Licensed under the Apache License, Version 2.0 (the "License");
5+ * you may not use this file except in compliance with the License.
6+ * You may obtain a copy of the License at
7+ *
8+ * http://www.apache.org/licenses/LICENSE-2.0
9+ *
10+ * Unless required by applicable law or agreed to in writing, software
11+ * distributed under the License is distributed on an "AS IS" BASIS,
12+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+ * See the License for the specific language governing permissions and
14+ * limitations under the License.
15+ */
16+
17+package s3select
18+
19+import (
20+ "bytes"
21+ "compress/bzip2"
22+ "encoding/csv"
23+ "encoding/xml"
24+ "io"
25+ "strconv"
26+ "strings"
27+ "time"
28+
29+ "net/http"
30+
31+ gzip "github.com/klauspost/pgzip"
32+)
33+
34+const (
35+ // progressTime is the time interval for which a progress message is sent.
36+ progressTime time.Duration = 60 * time.Second
37+ // continuationTime is the time interval for which a continuation message is
38+ // sent.
39+ continuationTime time.Duration = 5 * time.Second
40+)
41+
42+// progress represents a struct that represents the format for XML of the
43+// progress messages
44+type progress struct {
45+ BytesScanned int64 `xml:"BytesScanned"`
46+ BytesProcessed int64 `xml:"BytesProcessed"`
47+ BytesReturned int64 `xml:"BytesReturned"`
48+ Xmlns string `xml:"xmlns,attr"`
49+}
50+
51+// stats represents a struct that represents the format for XML of the stat
52+// messages
53+type stats struct {
54+ BytesScanned int64 `xml:"BytesScanned"`
55+ BytesProcessed int64 `xml:"BytesProcessed"`
56+ BytesReturned int64 `xml:"BytesReturned"`
57+ Xmlns string `xml:"xmlns,attr"`
58+}
59+
60+// StatInfo is a struct that represents the
61+type statInfo struct {
62+ BytesScanned int64
63+ BytesReturned int64
64+ BytesProcessed int64
65+}
66+
67+// Input represents a record producing input from a formatted file or pipe.
68+type Input struct {
69+ options *Options
70+ reader *csv.Reader
71+ firstRow []string
72+ header []string
73+ minOutputLength int
74+ stats *statInfo
75+}
76+
77+// Options options are passed to the underlying encoding/csv reader.
78+type Options struct {
79+ // HasHeader when true, will treat the first row as a header row.
80+ HasHeader bool
81+
82+ // FieldDelimiter is the string that fields are delimited by.
83+ FieldDelimiter string
84+
85+ // Comments is the string the first character of a line of
86+ // text matches the comment character.
87+ Comments string
88+
89+ // Name of the table that is used for querying
90+ Name string
91+
92+ // ReadFrom is where the data will be read from.
93+ ReadFrom io.Reader
94+
95+ // If true then we need to add gzip or bzip reader.
96+ // to extract the csv.
97+ Compressed string
98+
99+ // SQL expression meant to be evaluated.
100+ Expression string
101+
102+ // What the outputted CSV will be delimited by .
103+ OutputFieldDelimiter string
104+
105+ // Size of incoming object
106+ StreamSize int64
107+
108+ // Whether Header is "USE" or another
109+ HeaderOpt bool
110+}
111+
112+// NewInput sets up a new Input, the first row is read when this is run.
113+// If there is a problem with reading the first row, the error is returned.
114+// Otherwise, the returned reader can be reliably consumed with ReadRecord()
115+// until ReadRecord() returns nil.
116+func NewInput(opts *Options) (*Input, error) {
117+ myReader := opts.ReadFrom
118+ var tempBytesScanned int64
119+ tempBytesScanned = 0
120+ if opts.Compressed == "GZIP" {
121+ tempBytesScanned = opts.StreamSize
122+ var err error
123+ if myReader, err = gzip.NewReader(opts.ReadFrom); err != nil {
124+ return nil, ErrTruncatedInput
125+ }
126+ } else if opts.Compressed == "BZIP2" {
127+ tempBytesScanned = opts.StreamSize
128+ myReader = bzip2.NewReader(opts.ReadFrom)
129+ }
130+
131+ progress := &statInfo{
132+ BytesScanned: tempBytesScanned,
133+ BytesProcessed: 0,
134+ BytesReturned: 0,
135+ }
136+ reader := &Input{
137+ options: opts,
138+ reader: csv.NewReader(myReader),
139+ stats: progress,
140+ }
141+ reader.firstRow = nil
142+
143+ reader.reader.FieldsPerRecord = -1
144+ if reader.options.FieldDelimiter != "" {
145+ reader.reader.Comma = rune(reader.options.FieldDelimiter[0])
146+ }
147+
148+ if reader.options.Comments != "" {
149+ reader.reader.Comment = rune(reader.options.Comments[0])
150+ }
151+
152+ // QuoteCharacter - " (defaulted currently)
153+ reader.reader.LazyQuotes = true
154+
155+ if err := reader.readHeader(); err != nil {
156+ return nil, err
157+ }
158+
159+ return reader, nil
160+}
161+
162+// ReadRecord reads a single record from the . Always returns successfully.
163+// If the record is empty, an empty []string is returned.
164+// Record expand to match the current row size, adding blank fields as needed.
165+// Records never return less then the number of fields in the first row.
166+// Returns nil on EOF
167+// In the event of a parse error due to an invalid record, it is logged, and
168+// an empty []string is returned with the number of fields in the first row,
169+// as if the record were empty.
170+//
171+// In general, this is a very tolerant of problems reader.
172+func (reader *Input) ReadRecord() []string {
173+ var row []string
174+ var fileErr error
175+
176+ if reader.firstRow != nil {
177+ row = reader.firstRow
178+ reader.firstRow = nil
179+ return row
180+ }
181+
182+ row, fileErr = reader.reader.Read()
183+ emptysToAppend := reader.minOutputLength - len(row)
184+ if fileErr == io.EOF || fileErr == io.ErrClosedPipe {
185+ return nil
186+ } else if _, ok := fileErr.(*csv.ParseError); ok {
187+ emptysToAppend = reader.minOutputLength
188+ }
189+
190+ if emptysToAppend > 0 {
191+ for counter := 0; counter < emptysToAppend; counter++ {
192+ row = append(row, "")
193+ }
194+ }
195+
196+ return row
197+}
198+
199+// convertMySQL Replaces double quote escape for column names with backtick for
200+// the MySQL parser
201+func convertMySQL(random string) string {
202+ return strings.Replace(random, "\"", "`", len(random))
203+}
204+
205+// readHeader reads the header into the header variable if the header is present
206+// as the first row of the csv
207+func (reader *Input) readHeader() error {
208+ var readErr error
209+ if reader.options.HasHeader {
210+ reader.firstRow, readErr = reader.reader.Read()
211+ if readErr != nil {
212+ return ErrCSVParsingError
213+ }
214+ reader.header = reader.firstRow
215+ reader.firstRow = nil
216+ reader.minOutputLength = len(reader.header)
217+ } else {
218+ reader.firstRow, readErr = reader.reader.Read()
219+ reader.header = make([]string, len(reader.firstRow))
220+ for i := 0; i < reader.minOutputLength; i++ {
221+ reader.header[i] = strconv.Itoa(i)
222+ }
223+
224+ }
225+ return nil
226+}
227+
228+// createStatXML is the function which does the marshaling from the stat
229+// structs into XML so that the progress and stat message can be sent
230+func (reader *Input) createStatXML() (string, error) {
231+ if reader.options.Compressed == "NONE" {
232+ reader.stats.BytesProcessed = reader.options.StreamSize
233+ reader.stats.BytesScanned = reader.stats.BytesProcessed
234+ }
235+ statXML := stats{
236+ BytesScanned: reader.stats.BytesScanned,
237+ BytesProcessed: reader.stats.BytesProcessed,
238+ BytesReturned: reader.stats.BytesReturned,
239+ Xmlns: "",
240+ }
241+ out, err := xml.Marshal(statXML)
242+ if err != nil {
243+ return "", err
244+ }
245+ return xml.Header + string(out), nil
246+}
247+
248+// createProgressXML is the function which does the marshaling from the progress structs into XML so that the progress and stat message can be sent
249+func (reader *Input) createProgressXML() (string, error) {
250+ if reader.options.HasHeader {
251+ reader.stats.BytesProcessed += processSize(reader.header)
252+ }
253+ if !(reader.options.Compressed != "NONE") {
254+ reader.stats.BytesScanned = reader.stats.BytesProcessed
255+ }
256+ progressXML := &progress{
257+ BytesScanned: reader.stats.BytesScanned,
258+ BytesProcessed: reader.stats.BytesProcessed,
259+ BytesReturned: reader.stats.BytesReturned,
260+ Xmlns: "",
261+ }
262+ out, err := xml.Marshal(progressXML)
263+ if err != nil {
264+ return "", err
265+ }
266+ return xml.Header + string(out), nil
267+}
268+
269+// Header returns the header of the reader. Either the first row if a header
270+// set in the options, or c#, where # is the column number, starting with 0.
271+func (reader *Input) Header() []string {
272+ return reader.header
273+}
274+
275+// Row is a Struct for keeping track of key aspects of a row.
276+type Row struct {
277+ record string
278+ err error
279+}
280+
281+// Execute is the function where all the blocking occurs, It writes to the HTTP
282+// response writer in a streaming fashion so that the client can actively use
283+// the results before the query is finally finished executing. The
284+func (reader *Input) Execute(writer io.Writer) error {
285+ myRow := make(chan *Row)
286+ curBuf := bytes.NewBuffer(make([]byte, 1000000))
287+ curBuf.Reset()
288+ progressTicker := time.NewTicker(progressTime)
289+ continuationTimer := time.NewTimer(continuationTime)
290+ defer progressTicker.Stop()
291+ defer continuationTimer.Stop()
292+ go reader.runSelectParser(convertMySQL(reader.options.Expression), myRow)
293+ for {
294+ select {
295+ case row, ok := <-myRow:
296+ if ok && row.err != nil {
297+ errorMessage := reader.writeErrorMessage(row.err, curBuf)
298+ _, err := errorMessage.WriteTo(writer)
299+ flusher, okFlush := writer.(http.Flusher)
300+ if okFlush {
301+ flusher.Flush()
302+ }
303+ if err != nil {
304+ return err
305+ }
306+ curBuf.Reset()
307+ close(myRow)
308+ return nil
309+ } else if ok {
310+ message := reader.writeRecordMessage(row.record, curBuf)
311+ _, err := message.WriteTo(writer)
312+ flusher, okFlush := writer.(http.Flusher)
313+ if okFlush {
314+ flusher.Flush()
315+ }
316+ if err != nil {
317+ return err
318+ }
319+ curBuf.Reset()
320+ reader.stats.BytesReturned += int64(len(row.record))
321+ if !continuationTimer.Stop() {
322+ <-continuationTimer.C
323+ }
324+ continuationTimer.Reset(continuationTime)
325+ } else if !ok {
326+ statPayload, err := reader.createStatXML()
327+ if err != nil {
328+ return err
329+ }
330+ statMessage := reader.writeStatMessage(statPayload, curBuf)
331+ _, err = statMessage.WriteTo(writer)
332+ flusher, ok := writer.(http.Flusher)
333+ if ok {
334+ flusher.Flush()
335+ }
336+ if err != nil {
337+ return err
338+ }
339+ curBuf.Reset()
340+ message := reader.writeEndMessage(curBuf)
341+ _, err = message.WriteTo(writer)
342+ flusher, ok = writer.(http.Flusher)
343+ if ok {
344+ flusher.Flush()
345+ }
346+ if err != nil {
347+ return err
348+ }
349+ return nil
350+ }
351+
352+ case <-progressTicker.C:
353+ progressPayload, err := reader.createProgressXML()
354+ if err != nil {
355+ return err
356+ }
357+ progressMessage := reader.writeProgressMessage(progressPayload, curBuf)
358+ _, err = progressMessage.WriteTo(writer)
359+ flusher, ok := writer.(http.Flusher)
360+ if ok {
361+ flusher.Flush()
362+ }
363+ if err != nil {
364+ return err
365+ }
366+ curBuf.Reset()
367+ case <-continuationTimer.C:
368+ message := reader.writeContinuationMessage(curBuf)
369+ _, err := message.WriteTo(writer)
370+ flusher, ok := writer.(http.Flusher)
371+ if ok {
372+ flusher.Flush()
373+ }
374+ if err != nil {
375+ return err
376+ }
377+ curBuf.Reset()
378+ continuationTimer.Reset(continuationTime)
379+ }
380+ }
381+}
pkg/s3select/output.go+460−0
@@ -0,0 +1,460 @@
1+/*
2+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
3+ *
4+ * Licensed under the Apache License, Version 2.0 (the "License");
5+ * you may not use this file except in compliance with the License.
6+ * You may obtain a copy of the License at
7+ *
8+ * http://www.apache.org/licenses/LICENSE-2.0
9+ *
10+ * Unless required by applicable law or agreed to in writing, software
11+ * distributed under the License is distributed on an "AS IS" BASIS,
12+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+ * See the License for the specific language governing permissions and
14+ * limitations under the License.
15+ */
16+// DO NOT EDIT THIS PACKAGE DIRECTLY: This follows the protocol defined by
17+// AmazonS3 found at
18+// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
19+// Consult the Spec before making direct edits.
20+
21+package s3select
22+
23+import (
24+ "bytes"
25+ "encoding/binary"
26+ "hash/crc32"
27+)
28+
29+// Record Headers
30+// -11 -event type - 7 - 7 "Records"
31+// -13 -content-type -7 -24 "application/octet-stream"
32+// -13 -message-type -7 5 "event"
33+// This is predefined from AMZ protocol found here:
34+// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
35+var recordHeaders []byte
36+
37+// End Headers
38+// -13 -message-type -7 -5 "event"
39+// -11 -:event-type -7 -3 "End"
40+// This is predefined from AMZ protocol found here:
41+// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
42+var endHeaders []byte
43+
44+// Continuation Headers
45+// -13 -message-type -7 -5 "event"
46+// -11 -:event-type -7 -4 "Cont"
47+// This is predefined from AMZ protocol found here:
48+// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
49+var contHeaders []byte
50+
51+// Stat Headers
52+// -11 -event type - 7 - 5 "Stat" -20
53+// -13 -content-type -7 -8 "text/xml" -25
54+// -13 -message-type -7 -5 "event" -22
55+// This is predefined from AMZ protocol found here:
56+// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
57+var statHeaders []byte
58+
59+// Progress Headers
60+// -11 -event type - 7 - 8 "Progress" -23
61+// -13 -content-type -7 -8 "text/xml" -25
62+// -13 -message-type -7 -5 "event" -22
63+// This is predefined from AMZ protocol found here:
64+// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
65+var progressHeaders []byte
66+
67+// The length of the nonvariable portion of the ErrHeaders
68+// The below are the specifications of the header for a "error" event
69+// -11 -error-code - 7 - DEFINED "DEFINED"
70+// -14 -error-message -7 -DEFINED "DEFINED"
71+// -13 -message-type -7 -5 "error"
72+// This is predefined from AMZ protocol found here:
73+// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
74+var errHdrLen int
75+
76+func init() {
77+ recordHeaders = writeRecordHeader()
78+ endHeaders = writeEndHeader()
79+ contHeaders = writeContHeader()
80+ statHeaders = writeStatHeader()
81+ progressHeaders = writeProgressHeader()
82+ errHdrLen = 55
83+
84+}
85+
86+// encodeString encodes a string in a []byte, lenBytes is the number of bytes
87+// used to encode the length of the string.
88+func encodeHeaderStringValue(s string) []byte {
89+ n := uint16(len(s))
90+ lenSlice := make([]byte, 2)
91+ binary.BigEndian.PutUint16(lenSlice[0:], n)
92+ return append(lenSlice, []byte(s)...)
93+}
94+func encodeHeaderStringName(s string) []byte {
95+ lenSlice := make([]byte, 1)
96+ lenSlice[0] = byte(len(s))
97+ return append(lenSlice, []byte(s)...)
98+}
99+
100+// encodeNumber encodes a number in a []byte, lenBytes is the number of bytes
101+// used to encode the length of the string.
102+func encodeNumber(n byte, lenBytes int) []byte {
103+ lenSlice := make([]byte, lenBytes)
104+ lenSlice[0] = n
105+ return lenSlice
106+}
107+
108+// writePayloadSize writes the 4byte payload size portion of the protocol.
109+func writePayloadSize(payloadSize int, headerLength int) []byte {
110+ totalByteLen := make([]byte, 4)
111+ totalMsgLen := uint32(payloadSize + headerLength + 16)
112+ binary.BigEndian.PutUint32(totalByteLen, totalMsgLen)
113+ return totalByteLen
114+}
115+
116+// writeHeaderSize writes the 4byte header size portion of the protocol.
117+func writeHeaderSize(headerLength int) []byte {
118+ totalHeaderLen := make([]byte, 4)
119+ totalLen := uint32(headerLength)
120+ binary.BigEndian.PutUint32(totalHeaderLen, totalLen)
121+ return totalHeaderLen
122+}
123+
124+// writeCRC writes the CRC for both the prelude and and the end of the protocol.
125+func writeCRC(myBuffer []byte) []byte {
126+ // Calculate the CRC here:
127+ myCRC := make([]byte, 4)
128+ cksum := crc32.ChecksumIEEE(myBuffer)
129+ binary.BigEndian.PutUint32(myCRC, cksum)
130+ return myCRC
131+}
132+
133+// writePayload writes the Payload for those protocols which the Payload is
134+// necessary.
135+func writePayload(myPayload string) []byte {
136+ convertedPayload := []byte(myPayload)
137+ payloadStore := make([]byte, len(convertedPayload))
138+ copy(payloadStore[0:], myPayload)
139+ return payloadStore
140+}
141+
142+// writeRecordHeader is a function which writes the headers for the continuation
143+// Message
144+func writeRecordHeader() []byte {
145+ // This is predefined from AMZ protocol found here:
146+ // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
147+ var currentMessage = &bytes.Buffer{}
148+ // 11 -event type - 7 - 7 "Records"
149+ // header name
150+ currentMessage.Write(encodeHeaderStringName(":event-type"))
151+ // header type
152+ currentMessage.Write(encodeNumber(7, 1))
153+ // header value and header value length
154+ currentMessage.Write(encodeHeaderStringValue("Records"))
155+ // Creation of the Header for Content-Type // 13 -content-type -7 -24
156+ // "application/octet-stream"
157+ // header name
158+ currentMessage.Write(encodeHeaderStringName(":content-type"))
159+ // header type
160+ currentMessage.Write(encodeNumber(7, 1))
161+ // header value and header value length
162+ currentMessage.Write(encodeHeaderStringValue("application/octet-stream"))
163+ // Creation of the Header for message-type 13 -message-type -7 5 "event"
164+ // header name
165+ currentMessage.Write(encodeHeaderStringName(":message-type"))
166+ // header type
167+ currentMessage.Write(encodeNumber(7, 1))
168+ // header value and header value length
169+ currentMessage.Write(encodeHeaderStringValue("event"))
170+ return currentMessage.Bytes()
171+}
172+
173+// writeEndHeader is a function which writes the headers for the continuation
174+// Message
175+func writeEndHeader() []byte {
176+ // This is predefined from AMZ protocol found here:
177+ // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
178+ var currentMessage = &bytes.Buffer{}
179+ // header name
180+ currentMessage.Write(encodeHeaderStringName(":event-type"))
181+ // header type
182+ currentMessage.Write(encodeNumber(7, 1))
183+ // header value and header value length
184+ currentMessage.Write(encodeHeaderStringValue("End"))
185+
186+ // Creation of the Header for message-type 13 -message-type -7 5 "event"
187+ // header name
188+ currentMessage.Write(encodeHeaderStringName(":message-type"))
189+ // header type
190+ currentMessage.Write(encodeNumber(7, 1))
191+ // header value and header value length
192+ currentMessage.Write(encodeHeaderStringValue("event"))
193+ return currentMessage.Bytes()
194+}
195+
196+// writeContHeader is a function which writes the headers for the continuation
197+// Message
198+func writeContHeader() []byte {
199+ // This is predefined from AMZ protocol found here:
200+ // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
201+ var currentMessage = &bytes.Buffer{}
202+ // header name
203+ currentMessage.Write(encodeHeaderStringName(":event-type"))
204+ // header type
205+ currentMessage.Write(encodeNumber(7, 1))
206+ // header value and header value length
207+ currentMessage.Write(encodeHeaderStringValue("Cont"))
208+
209+ // Creation of the Header for message-type 13 -message-type -7 5 "event"
210+ // header name
211+ currentMessage.Write(encodeHeaderStringName(":message-type"))
212+ // header type
213+ currentMessage.Write(encodeNumber(7, 1))
214+ // header value and header value length
215+ currentMessage.Write(encodeHeaderStringValue("event"))
216+ return currentMessage.Bytes()
217+
218+}
219+
220+// writeStatHeader is a function which writes the headers for the Stat
221+// Message
222+func writeStatHeader() []byte {
223+ // This is predefined from AMZ protocol found here:
224+ // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
225+ var currentMessage = &bytes.Buffer{}
226+ // header name
227+ currentMessage.Write(encodeHeaderStringName(":event-type"))
228+ // header type
229+ currentMessage.Write(encodeNumber(7, 1))
230+ // header value and header value length
231+ currentMessage.Write(encodeHeaderStringValue("Stats"))
232+ // Creation of the Header for Content-Type // 13 -content-type -7 -8
233+ // "text/xml"
234+ // header name
235+ currentMessage.Write(encodeHeaderStringName(":content-type"))
236+ // header type
237+ currentMessage.Write(encodeNumber(7, 1))
238+ // header value and header value length
239+ currentMessage.Write(encodeHeaderStringValue("text/xml"))
240+
241+ // Creation of the Header for message-type 13 -message-type -7 5 "event"
242+ currentMessage.Write(encodeHeaderStringName(":message-type"))
243+ // header type
244+ currentMessage.Write(encodeNumber(7, 1))
245+ // header value and header value length
246+ currentMessage.Write(encodeHeaderStringValue("event"))
247+ return currentMessage.Bytes()
248+
249+}
250+
251+// writeProgressHeader is a function which writes the headers for the Progress
252+// Message
253+func writeProgressHeader() []byte {
254+ // This is predefined from AMZ protocol found here:
255+ // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
256+ var currentMessage = &bytes.Buffer{}
257+ // header name
258+ currentMessage.Write(encodeHeaderStringName(":event-type"))
259+ // header type
260+ currentMessage.Write(encodeNumber(7, 1))
261+ // header value and header value length
262+ currentMessage.Write(encodeHeaderStringValue("Progress"))
263+ // Creation of the Header for Content-Type // 13 -content-type -7 -8
264+ // "text/xml"
265+ // header name
266+ currentMessage.Write(encodeHeaderStringName(":content-type"))
267+ // header type
268+ currentMessage.Write(encodeNumber(7, 1))
269+ // header value and header value length
270+ currentMessage.Write(encodeHeaderStringValue("text/xml"))
271+
272+ // Creation of the Header for message-type 13 -message-type -7 5 "event"
273+ // header name
274+ currentMessage.Write(encodeHeaderStringName(":message-type"))
275+ // header type
276+ currentMessage.Write(encodeNumber(7, 1))
277+ // header value and header value length
278+ currentMessage.Write(encodeHeaderStringValue("event"))
279+ return currentMessage.Bytes()
280+
281+}
282+
283+// writeRecordMessage is the function which constructs the binary message for a
284+// record message to be sent.
285+func (csvOutput *Input) writeRecordMessage(payload string, currentMessage *bytes.Buffer) *bytes.Buffer {
286+ // The below are the specifications of the header for a "record" event
287+ // 11 -event type - 7 - 7 "Records"
288+ // 13 -content-type -7 -24 "application/octet-stream"
289+ // 13 -message-type -7 5 "event"
290+ // This is predefined from AMZ protocol found here:
291+ // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
292+ headerLen := len(recordHeaders)
293+ // Writes the total size of the message.
294+ currentMessage.Write(writePayloadSize(len(payload), headerLen))
295+ // Writes the total size of the header.
296+ currentMessage.Write(writeHeaderSize(headerLen))
297+ // Writes the CRC of the Prelude
298+ currentMessage.Write(writeCRC(currentMessage.Bytes()))
299+ currentMessage.Write(recordHeaders)
300+
301+ // This part is where the payload is written, this will be only one row, since
302+ // we're sending one message at a types
303+ currentMessage.Write(writePayload(payload))
304+
305+ // Now we do a CRC check on the entire messages
306+ currentMessage.Write(writeCRC(currentMessage.Bytes()))
307+ return currentMessage
308+
309+}
310+
311+// writeContinuationMessage is the function which constructs the binary message
312+// for a continuation message to be sent.
313+func (csvOutput *Input) writeContinuationMessage(currentMessage *bytes.Buffer) *bytes.Buffer {
314+ // 11 -event type - 7 - 4 "Cont"
315+ // 13 -message-type -7 5 "event"
316+ // This is predefined from AMZ protocol found here:
317+ // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
318+ headerLen := len(contHeaders)
319+ currentMessage.Write(writePayloadSize(0, headerLen))
320+
321+ currentMessage.Write(writeHeaderSize(headerLen))
322+
323+ // Calculate the Prelude CRC here:
324+ currentMessage.Write(writeCRC(currentMessage.Bytes()))
325+
326+ currentMessage.Write(contHeaders)
327+
328+ //Now we do a CRC check on the entire messages
329+ currentMessage.Write(writeCRC(currentMessage.Bytes()))
330+ return currentMessage
331+
332+}
333+
334+// writeEndMessage is the function which constructs the binary message
335+// for a end message to be sent.
336+func (csvOutput *Input) writeEndMessage(currentMessage *bytes.Buffer) *bytes.Buffer {
337+ // 11 -event type - 7 - 3 "End"
338+ // 13 -message-type -7 5 "event"
339+ // This is predefined from AMZ protocol found here:
340+ // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
341+ headerLen := len(endHeaders)
342+ currentMessage.Write(writePayloadSize(0, headerLen))
343+
344+ currentMessage.Write(writeHeaderSize(headerLen))
345+
346+ //Calculate the Prelude CRC here:
347+ currentMessage.Write(writeCRC(currentMessage.Bytes()))
348+
349+ currentMessage.Write(endHeaders)
350+
351+ // Now we do a CRC check on the entire messages
352+ currentMessage.Write(writeCRC(currentMessage.Bytes()))
353+ return currentMessage
354+
355+}
356+
357+// writeStateMessage is the function which constructs the binary message for a
358+// state message to be sent.
359+func (csvOutput *Input) writeStatMessage(payload string, currentMessage *bytes.Buffer) *bytes.Buffer {
360+ // 11 -event type - 7 - 5 "Stat" 20
361+ // 13 -content-type -7 -8 "text/xml" 25
362+ // 13 -message-type -7 5 "event" 22
363+ // This is predefined from AMZ protocol found here:
364+ // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
365+ headerLen := len(statHeaders)
366+
367+ currentMessage.Write(writePayloadSize(len(payload), headerLen))
368+
369+ currentMessage.Write(writeHeaderSize(headerLen))
370+
371+ currentMessage.Write(writeCRC(currentMessage.Bytes()))
372+
373+ currentMessage.Write(statHeaders)
374+
375+ // This part is where the payload is written, this will be only one row, since
376+ // we're sending one message at a types
377+ currentMessage.Write(writePayload(payload))
378+
379+ // Now we do a CRC check on the entire messages
380+ currentMessage.Write(writeCRC(currentMessage.Bytes()))
381+ return currentMessage
382+
383+}
384+
385+// writeProgressMessage is the function which constructs the binary message for
386+// a progress message to be sent.
387+func (csvOutput *Input) writeProgressMessage(payload string, currentMessage *bytes.Buffer) *bytes.Buffer {
388+ // The below are the specifications of the header for a "Progress" event
389+ // 11 -event type - 7 - 8 "Progress" 23
390+ // 13 -content-type -7 -8 "text/xml" 25
391+ // 13 -message-type -7 5 "event" 22
392+ // This is predefined from AMZ protocol found here:
393+ // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
394+ headerLen := len(progressHeaders)
395+
396+ currentMessage.Write(writePayloadSize(len(payload), headerLen))
397+
398+ currentMessage.Write(writeHeaderSize(headerLen))
399+
400+ currentMessage.Write(writeCRC(currentMessage.Bytes()))
401+
402+ currentMessage.Write(progressHeaders)
403+
404+ // This part is where the payload is written, this will be only one row, since
405+ // we're sending one message at a types
406+ currentMessage.Write(writePayload(payload))
407+
408+ // Now we do a CRC check on the entire messages
409+ currentMessage.Write(writeCRC(currentMessage.Bytes()))
410+ return currentMessage
411+
412+}
413+
414+// writeErrorMessage is the function which constructs the binary message for a
415+// error message to be sent.
416+func (csvOutput *Input) writeErrorMessage(errorMessage error, currentMessage *bytes.Buffer) *bytes.Buffer {
417+
418+ // The below are the specifications of the header for a "error" event
419+ // 11 -error-code - 7 - DEFINED "DEFINED"
420+ // 14 -error-message -7 -DEFINED "DEFINED"
421+ // 13 -message-type -7 5 "error"
422+ // This is predefined from AMZ protocol found here:
423+ // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html
424+ sizeOfErrorCode := len(errorCodeResponse[errorMessage])
425+ sizeOfErrorMessage := len(errorMessage.Error())
426+ headerLen := errHdrLen + sizeOfErrorCode + sizeOfErrorMessage
427+
428+ currentMessage.Write(writePayloadSize(0, headerLen))
429+
430+ currentMessage.Write(writeHeaderSize(headerLen))
431+
432+ currentMessage.Write(writeCRC(currentMessage.Bytes()))
433+ // header name
434+ currentMessage.Write(encodeHeaderStringName(":error-code"))
435+ // header type
436+ currentMessage.Write(encodeNumber(7, 1))
437+ // header value and header value length
438+ currentMessage.Write(encodeHeaderStringValue(errorCodeResponse[errorMessage]))
439+
440+ // 14 -error-message -7 -DEFINED "DEFINED"
441+
442+ // header name
443+ currentMessage.Write(encodeHeaderStringName(":error-message"))
444+ // header type
445+ currentMessage.Write(encodeNumber(7, 1))
446+ // header value and header value length
447+ currentMessage.Write(encodeHeaderStringValue(errorMessage.Error()))
448+ // Creation of the Header for message-type 13 -message-type -7 5 "error"
449+ // header name
450+ currentMessage.Write(encodeHeaderStringName(":message-type"))
451+ // header type
452+ currentMessage.Write(encodeNumber(7, 1))
453+ // header value and header value length
454+ currentMessage.Write(encodeHeaderStringValue("error"))
455+
456+ // Now we do a CRC check on the entire messages
457+ currentMessage.Write(writeCRC(currentMessage.Bytes()))
458+ return currentMessage
459+
460+}
pkg/s3select/select.go+415−0
@@ -0,0 +1,415 @@
1+/*
2+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
3+ *
4+ * Licensed under the Apache License, Version 2.0 (the "License");
5+ * you may not use this file except in compliance with the License.
6+ * You may obtain a copy of the License at
7+ *
8+ * http://www.apache.org/licenses/LICENSE-2.0
9+ *
10+ * Unless required by applicable law or agreed to in writing, software
11+ * distributed under the License is distributed on an "AS IS" BASIS,
12+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+ * See the License for the specific language governing permissions and
14+ * limitations under the License.
15+ */
16+
17+package s3select
18+
19+import (
20+ "math"
21+ "strconv"
22+ "strings"
23+
24+ "github.com/xwb1989/sqlparser"
25+)
26+
27+// SelectFuncs contains the relevant values from the parser for S3 Select
28+// Functions
29+type SelectFuncs struct {
30+ funcExpr []*sqlparser.FuncExpr
31+ index []int
32+}
33+
34+// RunSqlParser allows us to easily bundle all the functions from above and run
35+// them in the appropriate order.
36+func (reader *Input) runSelectParser(selectExpression string, myRow chan *Row) {
37+ reqCols, alias, myLimit, whereClause, aggFunctionNames, myFuncs, myErr := reader.ParseSelect(selectExpression)
38+ if myErr != nil {
39+ rowStruct := &Row{
40+ err: myErr,
41+ }
42+ myRow <- rowStruct
43+ return
44+ }
45+ reader.processSelectReq(reqCols, alias, whereClause, myLimit, aggFunctionNames, myRow, myFuncs)
46+}
47+
48+// ParseSelect parses the SELECT expression, and effectively tokenizes it into
49+// its separate parts. It returns the requested column names,alias,limit of
50+// records, and the where clause.
51+func (reader *Input) ParseSelect(sqlInput string) ([]string, string, int, interface{}, []string, *SelectFuncs, error) {
52+ // return columnNames, alias, limitOfRecords, whereclause,coalStore, nil
53+
54+ stmt, err := sqlparser.Parse(sqlInput)
55+ var whereClause interface{}
56+ var alias string
57+ var limit int
58+ myFuncs := &SelectFuncs{}
59+ // TODO Maybe can parse their errors a bit to return some more of the s3 errors
60+ if err != nil {
61+ return nil, "", 0, nil, nil, nil, ErrLexerInvalidChar
62+ }
63+ switch stmt := stmt.(type) {
64+ case *sqlparser.Select:
65+ // evaluates the where clause
66+ functionNames := make([]string, len(stmt.SelectExprs))
67+ columnNames := make([]string, len(stmt.SelectExprs))
68+
69+ if stmt.Where != nil {
70+ switch expr := stmt.Where.Expr.(type) {
71+ default:
72+ whereClause = expr
73+ case *sqlparser.ComparisonExpr:
74+ whereClause = expr
75+ }
76+ }
77+ if stmt.SelectExprs != nil {
78+ for i := 0; i < len(stmt.SelectExprs); i++ {
79+ switch expr := stmt.SelectExprs[i].(type) {
80+ case *sqlparser.StarExpr:
81+ columnNames[0] = "*"
82+ case *sqlparser.AliasedExpr:
83+ switch smallerexpr := expr.Expr.(type) {
84+ case *sqlparser.FuncExpr:
85+ if smallerexpr.IsAggregate() {
86+ functionNames[i] = smallerexpr.Name.CompliantName()
87+ // Will return function name
88+ // Case to deal with if we have functions and not an asterix
89+ switch tempagg := smallerexpr.Exprs[0].(type) {
90+ case *sqlparser.StarExpr:
91+ columnNames[0] = "*"
92+ if smallerexpr.Name.CompliantName() != "count" {
93+ return nil, "", 0, nil, nil, nil, ErrParseUnsupportedCallWithStar
94+ }
95+ case *sqlparser.AliasedExpr:
96+ switch col := tempagg.Expr.(type) {
97+ case *sqlparser.BinaryExpr:
98+ return nil, "", 0, nil, nil, nil, ErrParseNonUnaryAgregateFunctionCall
99+ case *sqlparser.ColName:
100+ columnNames[i] = col.Name.CompliantName()
101+ }
102+ }
103+ // Case to deal with if COALESCE was used..
104+ } else if supportedFunc(smallerexpr.Name.CompliantName()) {
105+ if myFuncs.funcExpr == nil {
106+ myFuncs.funcExpr = make([]*sqlparser.FuncExpr, len(stmt.SelectExprs))
107+ myFuncs.index = make([]int, len(stmt.SelectExprs))
108+ }
109+ myFuncs.funcExpr[i] = smallerexpr
110+ myFuncs.index[i] = i
111+ } else {
112+ return nil, "", 0, nil, nil, nil, ErrUnsupportedSQLOperation
113+ }
114+ case *sqlparser.ColName:
115+ columnNames[i] = smallerexpr.Name.CompliantName()
116+ }
117+ }
118+ }
119+ }
120+
121+ // This code retrieves the alias and makes sure it is set to the correct
122+ // value, if not it sets it to the tablename
123+ if (stmt.From) != nil {
124+ for i := 0; i < len(stmt.From); i++ {
125+ switch smallerexpr := stmt.From[i].(type) {
126+ case *sqlparser.JoinTableExpr:
127+ return nil, "", 0, nil, nil, nil, ErrParseMalformedJoin
128+ case *sqlparser.AliasedTableExpr:
129+ alias = smallerexpr.As.CompliantName()
130+ if alias == "" {
131+ alias = sqlparser.GetTableName(smallerexpr.Expr).CompliantName()
132+ }
133+ }
134+ }
135+ }
136+ if stmt.Limit != nil {
137+ switch expr := stmt.Limit.Rowcount.(type) {
138+ case *sqlparser.SQLVal:
139+ // The Value of how many rows we're going to limit by
140+ limit, _ = strconv.Atoi(string(expr.Val[:]))
141+ }
142+ }
143+ if stmt.GroupBy != nil {
144+ return nil, "", 0, nil, nil, nil, ErrParseUnsupportedLiteralsGroupBy
145+ }
146+ if stmt.OrderBy != nil {
147+ return nil, "", 0, nil, nil, nil, ErrParseUnsupportedToken
148+ }
149+ if err := reader.parseErrs(columnNames, whereClause, alias, myFuncs); err != nil {
150+ return nil, "", 0, nil, nil, nil, err
151+ }
152+ return columnNames, alias, limit, whereClause, functionNames, myFuncs, nil
153+ }
154+ return nil, "", 0, nil, nil, nil, nil
155+}
156+
157+// This is the main function, It goes row by row and for records which validate
158+// the where clause it currently prints the appropriate row given the requested
159+// columns.
160+func (reader *Input) processSelectReq(reqColNames []string, alias string, whereClause interface{}, limitOfRecords int, functionNames []string, myRow chan *Row, myFunc *SelectFuncs) {
161+ counter := -1
162+ filtrCount := 0
163+ functionFlag := false
164+ // My values is used to store our aggregation values if we need to store them.
165+ myAggVals := make([]float64, len(reqColNames))
166+ var columns []string
167+ // LowercasecolumnsMap is used in accordance with hasDuplicates so that we can
168+ // raise the error "Ambigious" if a case insensitive column is provided and we
169+ // have multiple matches.
170+ lowercaseColumnsMap := make(map[string]int)
171+ hasDuplicates := make(map[string]bool)
172+ // ColumnsMap stores our columns and their index.
173+ columnsMap := make(map[string]int)
174+ if limitOfRecords == 0 {
175+ limitOfRecords = math.MaxInt64
176+ }
177+
178+ for {
179+ record := reader.ReadRecord()
180+ reader.stats.BytesProcessed += processSize(record)
181+ if record == nil {
182+ if functionFlag {
183+ rowStruct := &Row{
184+ record: reader.aggFuncToStr(myAggVals) + "\n",
185+ }
186+ myRow <- rowStruct
187+ }
188+ close(myRow)
189+ return
190+ }
191+ if counter == -1 && reader.options.HeaderOpt && len(reader.header) > 0 {
192+ columns = reader.Header()
193+ myErr := checkForDuplicates(columns, columnsMap, hasDuplicates, lowercaseColumnsMap)
194+ if myErr != nil {
195+ rowStruct := &Row{
196+ err: myErr,
197+ }
198+ myRow <- rowStruct
199+ return
200+ }
201+ } else if counter == -1 && len(reader.header) > 0 {
202+ columns = reader.Header()
203+ }
204+ // When we have reached our limit, on what the user specified as the number
205+ // of rows they wanted, we terminate our interpreter.
206+ if filtrCount == limitOfRecords && limitOfRecords != 0 {
207+ close(myRow)
208+ return
209+ }
210+ // The call to the where function clause,ensures that the rows we print match our where clause.
211+ condition, myErr := matchesMyWhereClause(record, columnsMap, alias, whereClause)
212+ if myErr != nil {
213+ rowStruct := &Row{
214+ err: myErr,
215+ }
216+ myRow <- rowStruct
217+ return
218+ }
219+ if condition {
220+ // if its an asterix we just print everything in the row
221+ if reqColNames[0] == "*" && functionNames[0] == "" {
222+ rowStruct := &Row{
223+ record: reader.printAsterix(record) + "\n",
224+ }
225+ myRow <- rowStruct
226+ } else if alias != "" {
227+ // This is for dealing with the case of if we have to deal with a
228+ // request for a column with an index e.g A_1.
229+ if representsInt(reqColNames[0]) {
230+ // This checks whether any aggregation function was called as now we
231+ // no longer will go through printing each row, and only print at the
232+ // end
233+ if len(functionNames) > 0 && functionNames[0] != "" {
234+ functionFlag = true
235+ aggregationFunctions(counter, filtrCount, myAggVals, columnsMap, reqColNames, functionNames, record)
236+ } else {
237+ // The code below finds the appropriate columns of the row given the
238+ // indicies provided in the SQL request and utilizes the map to
239+ // retrieve the correct part of the row.
240+ myQueryRow, myErr := reader.processColNameIndex(record, reqColNames, columns)
241+ if myErr != nil {
242+ rowStruct := &Row{
243+ err: myErr,
244+ }
245+ myRow <- rowStruct
246+ return
247+ }
248+ rowStruct := &Row{
249+ record: myQueryRow + "\n",
250+ }
251+ myRow <- rowStruct
252+ }
253+ } else {
254+ // This code does aggregation if we were provided column names in the
255+ // form of acutal names rather an indices.
256+ if len(functionNames) > 0 && functionNames[0] != "" {
257+ functionFlag = true
258+ aggregationFunctions(counter, filtrCount, myAggVals, columnsMap, reqColNames, functionNames, record)
259+ } else {
260+ // This code prints the appropriate part of the row given the filter
261+ // and select request, if the select request was based on column
262+ // names rather than indices.
263+ myQueryRow, myErr := reader.processColNameLiteral(record, reqColNames, columns, columnsMap, myFunc)
264+ if myErr != nil {
265+ rowStruct := &Row{
266+ err: myErr,
267+ }
268+ myRow <- rowStruct
269+ return
270+ }
271+ rowStruct := &Row{
272+ record: myQueryRow + "\n",
273+ }
274+ myRow <- rowStruct
275+ }
276+ }
277+ }
278+ filtrCount++
279+ }
280+ counter++
281+ }
282+}
283+
284+// printAsterix helps to print out the entire row if an asterix is used.
285+func (reader *Input) printAsterix(record []string) string {
286+ myRow := record[0]
287+ for i := 1; i < len(record); i++ {
288+ myRow = myRow + reader.options.OutputFieldDelimiter + record[i]
289+ }
290+ return myRow
291+}
292+
293+// processColumnNames is a function which allows for cleaning of column names.
294+func (reader *Input) processColumnNames(reqColNames []string, alias string) error {
295+ for i := 0; i < len(reqColNames); i++ {
296+ // The code below basically cleans the column name of its alias and other
297+ // syntax, so that we can extract its pure name.
298+ reqColNames[i] = cleanCol(reqColNames[i], alias)
299+ }
300+ return nil
301+}
302+
303+// processColNameIndex is the function which creates the row for an index based
304+// query.
305+func (reader *Input) processColNameIndex(record []string, reqColNames []string, columns []string) (string, error) {
306+ myRow := ""
307+ for i := 0; i < len(reqColNames); i++ {
308+ // COALESCE AND NULLIF do not support index based access.
309+ if reqColNames[0] == "0" {
310+ return "", ErrInvalidColumnIndex
311+ }
312+ // Subtract 1 because AWS Indexing is not 0 based, it starts at 1.
313+ mytempindex, err := strconv.Atoi(reqColNames[i])
314+ mytempindex = mytempindex - 1
315+ if mytempindex > len(columns) {
316+ return "", ErrInvalidColumnIndex
317+ }
318+ myRow = writeRow(myRow, record[mytempindex], reader.options.OutputFieldDelimiter, len(reqColNames))
319+ if err != nil {
320+ return "", ErrMissingHeaders
321+ }
322+ }
323+ if len(myRow) > 1000000 {
324+ return "", ErrOverMaxRecordSize
325+ }
326+ if strings.Count(myRow, reader.options.OutputFieldDelimiter) != len(reqColNames)-1 {
327+ myRow = qualityCheck(myRow, len(reqColNames)-1-strings.Count(myRow, reader.options.OutputFieldDelimiter), reader.options.OutputFieldDelimiter)
328+ }
329+ return myRow, nil
330+}
331+
332+// processColNameLiteral is the function which creates the row for an name based
333+// query.
334+func (reader *Input) processColNameLiteral(record []string, reqColNames []string, columns []string, columnsMap map[string]int, myFunc *SelectFuncs) (string, error) {
335+ myRow := ""
336+ for i := 0; i < len(reqColNames); i++ {
337+ // this is the case to deal with COALESCE.
338+ if reqColNames[i] == "" && isValidFunc(myFunc.index, i) {
339+ myVal := evaluateFuncExpr(myFunc.funcExpr[i], "", record, columnsMap)
340+ myRow = writeRow(myRow, myVal, reader.options.OutputFieldDelimiter, len(reqColNames))
341+ continue
342+ }
343+ myTempIndex, notFound := columnsMap[trimQuotes(reqColNames[i])]
344+ if !notFound {
345+ return "", ErrMissingHeaders
346+ }
347+ myRow = writeRow(myRow, record[myTempIndex], reader.options.OutputFieldDelimiter, len(reqColNames))
348+ }
349+ if len(myRow) > 1000000 {
350+ return "", ErrOverMaxRecordSize
351+ }
352+ if strings.Count(myRow, reader.options.OutputFieldDelimiter) != len(reqColNames)-1 {
353+ myRow = qualityCheck(myRow, len(reqColNames)-1-strings.Count(myRow, reader.options.OutputFieldDelimiter), reader.options.OutputFieldDelimiter)
354+ }
355+ return myRow, nil
356+
357+}
358+
359+// aggregationFunctions is a function which performs the actual aggregation
360+// methods on the given row, it uses an array defined the the main parsing
361+// function to keep track of values.
362+func aggregationFunctions(counter int, filtrCount int, myAggVals []float64, columnsMap map[string]int, storeReqCols []string, storeFunctions []string, record []string) error {
363+ for i := 0; i < len(storeFunctions); i++ {
364+ if storeFunctions[i] == "" {
365+ i++
366+ } else if storeFunctions[i] == "count" {
367+ myAggVals[i]++
368+ } else {
369+ // If column names are provided as an index it'll use this if statement instead of the else/
370+ var convAggFloat float64
371+ if representsInt(storeReqCols[i]) {
372+ myIndex, _ := strconv.Atoi(storeReqCols[i])
373+ convAggFloat, _ = strconv.ParseFloat(record[myIndex], 64)
374+
375+ } else {
376+ // case that the columns are in the form of named columns rather than indices.
377+ convAggFloat, _ = strconv.ParseFloat(record[columnsMap[trimQuotes(storeReqCols[i])]], 64)
378+
379+ }
380+ // This if statement is for calculating the min.
381+ if storeFunctions[i] == "min" {
382+ if counter == -1 {
383+ myAggVals[i] = math.MaxFloat64
384+ }
385+ if convAggFloat < myAggVals[i] {
386+ myAggVals[i] = convAggFloat
387+ }
388+
389+ } else if storeFunctions[i] == "max" {
390+ // This if statement is for calculating the max.
391+ if counter == -1 {
392+ myAggVals[i] = math.SmallestNonzeroFloat64
393+ }
394+ if convAggFloat > myAggVals[i] {
395+ myAggVals[i] = convAggFloat
396+ }
397+
398+ } else if storeFunctions[i] == "sum" {
399+ // This if statement is for calculating the sum.
400+ myAggVals[i] += convAggFloat
401+
402+ } else if storeFunctions[i] == "avg" {
403+ // This if statement is for calculating the average.
404+ if filtrCount == 0 {
405+ myAggVals[i] = convAggFloat
406+ } else {
407+ myAggVals[i] = (convAggFloat + (myAggVals[i] * float64(filtrCount))) / float64((filtrCount + 1))
408+ }
409+ } else {
410+ return ErrParseNonUnaryAgregateFunctionCall
411+ }
412+ }
413+ }
414+ return nil
415+}
pkg/s3select/select_test.go+1070−0
@@ -0,0 +1,1070 @@
1+/*
2+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
3+ *
4+ * Licensed under the Apache License, Version 2.0 (the "License");
5+ * you may not use this file except in compliance with the License.
6+ * You may obtain a copy of the License at
7+ *
8+ * http://www.apache.org/licenses/LICENSE-2.0
9+ *
10+ * Unless required by applicable law or agreed to in writing, software
11+ * distributed under the License is distributed on an "AS IS" BASIS,
12+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+ * See the License for the specific language governing permissions and
14+ * limitations under the License.
15+ */
16+
17+package s3select
18+
19+import (
20+ "bytes"
21+ "fmt"
22+ "reflect"
23+ "testing"
24+)
25+
26+// Unit Test for the checkForDuplicates function.
27+func TestCheckForDuplicates(t *testing.T) {
28+ tables := []struct {
29+ myReq []string
30+ myHeaders map[string]int
31+ myDup map[string]bool
32+ myLow map[string]int
33+ myErr error
34+ }{
35+ {[]string{"name", "id", "last_name", "last_name"}, make(map[string]int), make(map[string]bool), make(map[string]int), ErrAmbiguousFieldName},
36+ {[]string{"name", "id", "last_name", "another_name"}, make(map[string]int), make(map[string]bool), make(map[string]int), nil},
37+ }
38+ for _, table := range tables {
39+ err := checkForDuplicates(table.myReq, table.myHeaders, table.myDup, table.myLow)
40+ if err != table.myErr {
41+ t.Error()
42+ }
43+ }
44+}
45+
46+// Test for the function which processes columnnames to make sure that they are
47+// compatible with spaces.
48+func TestMyProcessing(t *testing.T) {
49+ options := &Options{
50+ HasHeader: false,
51+ FieldDelimiter: ",",
52+ Comments: "",
53+ Name: "S3Object", // Default table name for all objects
54+ ReadFrom: bytes.NewReader([]byte("Here , is, a, string + \n + random,random,stuff,stuff ")),
55+ Compressed: "",
56+ Expression: "",
57+ OutputFieldDelimiter: ",",
58+ StreamSize: 20,
59+ }
60+ s3s, err := NewInput(options)
61+ if err != nil {
62+ t.Error(err)
63+ }
64+ tables := []struct {
65+ myReq []string
66+ myHeaders map[string]int
67+ myDup map[string]bool
68+ myLow map[string]int
69+ myOpts *Options
70+ input *Input
71+ length int
72+ testOutput string
73+ myErr error
74+ }{
75+ {[]string{"name", "id", "last_name", "CAST"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, s3s, 4, "CAST", nil},
76+ {[]string{"name", "id", "last_name", "another_name"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, s3s, 4, "another_name", nil},
77+ {[]string{"name", "id", "last_name", "another_name"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, s3s, 4, "another_name", nil},
78+ {[]string{"name", "id", "random_name", "fame_name", "another_col"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, s3s, 5, "fame_name", nil},
79+ }
80+ for _, table := range tables {
81+ err = checkForDuplicates(table.myReq, table.myHeaders, table.myDup, table.myLow)
82+ if err != table.myErr {
83+ t.Error()
84+ }
85+ if len(table.myReq) != table.length {
86+ t.Errorf("UnexpectedError")
87+ }
88+ if table.myReq[3] != table.testOutput {
89+ t.Error()
90+ }
91+ }
92+}
93+
94+// TestMyRowIndexResults is a unit test which makes sure that the rows that are
95+// being printed are appropriate to the query being requested.
96+func TestMyRowIndexResults(t *testing.T) {
97+ options := &Options{
98+ HasHeader: false,
99+ FieldDelimiter: ",",
100+ Comments: "",
101+ Name: "S3Object", // Default table name for all objects
102+ ReadFrom: bytes.NewReader([]byte("Here , is, a, string + \n + random,random,stuff,stuff ")),
103+ Compressed: "",
104+ Expression: "",
105+ OutputFieldDelimiter: ",",
106+ StreamSize: 20,
107+ }
108+ s3s, err := NewInput(options)
109+ if err != nil {
110+ t.Error(err)
111+ }
112+ tables := []struct {
113+ myReq []string
114+ myHeaders map[string]int
115+ myDup map[string]bool
116+ myLow map[string]int
117+ myOpts *Options
118+ input *Input
119+ myRecord []string
120+ myTarget string
121+ myAsterix string
122+ columns []string
123+ err error
124+ }{
125+ {[]string{"1", "2"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, s3s, []string{"target", "random", "hello", "stuff"}, "target,random", "target,random,hello,stuff", []string{"1", "2", "3", "4"}, nil},
126+ {[]string{"2", "3", "4"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, s3s, []string{"random", "hullo", "thing", "stuff"}, "hullo,thing,stuff", "random,hullo,thing,stuff", []string{"1", "2", "3", "4"}, nil},
127+ {[]string{"3", "2"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, s3s, []string{"random", "hullo", "thing", "stuff"}, "thing,hullo", "random,hullo,thing,stuff", []string{"1", "2", "3", "4"}, nil},
128+ {[]string{"11", "1"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, s3s, []string{"random", "hullo", "thing", "stuff"}, "", "random,hullo,thing,stuff", []string{"1", "2", "3", "4"}, ErrInvalidColumnIndex},
129+ }
130+ for _, table := range tables {
131+ checkForDuplicates(table.columns, table.myHeaders, table.myDup, table.myLow)
132+ myRow, err := s3s.processColNameIndex(table.myRecord, table.myReq, table.columns)
133+ if err != table.err {
134+ t.Error()
135+ }
136+ if myRow != table.myTarget {
137+ t.Error()
138+ }
139+ myRow = table.input.printAsterix(table.myRecord)
140+ if myRow != table.myAsterix {
141+ t.Error()
142+ }
143+ }
144+}
145+
146+// TestMyHelperFunctions is a unit test which tests some small helper string
147+// functions.
148+func TestMyHelperFunctions(t *testing.T) {
149+ tables := []struct {
150+ myReq string
151+ myList []string
152+ myIndex int
153+ expected bool
154+ }{
155+ {"test1", []string{"test1", "test2", "test3", "test4", "test5"}, 0, true},
156+ {"random", []string{"test1", "test2", "test3", "test4", "test5"}, -1, false},
157+ {"test3", []string{"test1", "test2", "test3", "test4", "test5"}, 2, true},
158+ }
159+ for _, table := range tables {
160+ if stringInSlice(table.myReq, table.myList) != table.expected {
161+ t.Error()
162+ }
163+ if stringIndex(table.myReq, table.myList) != table.myIndex {
164+ t.Error()
165+ }
166+ }
167+}
168+
169+// TestMyStateMachine is a unit test which ensures that the lowest level of the
170+// interpreter is converting properly.
171+func TestMyStateMachine(t *testing.T) {
172+ tables := []struct {
173+ operand interface{}
174+ operator string
175+ leftArg string
176+ err error
177+ expected bool
178+ }{
179+ {"2005", ">", "2012", nil, true},
180+ {2005, ">", "2012", nil, true},
181+ {2012.0000, ">", "2014.000", nil, true},
182+ {"NA", ">", "2014.000", nil, false},
183+ {2014, ">", "Random", nil, false},
184+ {"test3", ">", "aandom", nil, false},
185+ }
186+ for _, table := range tables {
187+ val, err := evaluateOperator(table.leftArg, table.operator, table.operand)
188+ if err != table.err {
189+ t.Error()
190+ }
191+ if val != table.expected {
192+ t.Error()
193+ }
194+ }
195+}
196+
197+// TestMyOperators is a unit test which ensures that the appropriate values are
198+// being returned from the operators functions.
199+func TestMyOperators(t *testing.T) {
200+ tables := []struct {
201+ operator string
202+ err error
203+ }{
204+ {">", nil},
205+ {"%", ErrParseUnknownOperator},
206+ }
207+ for _, table := range tables {
208+ err := checkValidOperator(table.operator)
209+ if err != table.err {
210+ t.Error()
211+ }
212+ }
213+}
214+
215+// TestMyConversion ensures that the conversion of the value from the csv
216+// happens correctly.
217+func TestMyConversion(t *testing.T) {
218+ tables := []struct {
219+ myTblVal string
220+ expected reflect.Kind
221+ }{
222+ {"2014", reflect.Int},
223+ {"2014.000", reflect.Float64},
224+ {"String!!!", reflect.String},
225+ }
226+ for _, table := range tables {
227+ val := reflect.ValueOf(checkStringType(table.myTblVal)).Kind()
228+ if val != table.expected {
229+ t.Error()
230+ }
231+ }
232+}
233+
234+// Unit Tests for Parser.
235+func TestMyParser(t *testing.T) {
236+ tables := []struct {
237+ myQuery string
238+ err error
239+ reqCols []string
240+ alias string
241+ myLimit int
242+ aggFuncs []string
243+ header []string
244+ }{
245+ {"SELECT * FROM S3OBJECT", nil, []string{"*"}, "S3OBJECT", 0, make([]string, 1), []string{"name1", "name2", "name3", "name4"}},
246+ {"SELECT * FROM S3OBJECT AS A", nil, []string{"*"}, "A", 0, make([]string, 1), []string{"name1", "name2", "name3", "name4"}},
247+ {"SELECT col_name FROM S3OBJECT AS A", nil, []string{"col_name"}, "A", 0, make([]string, 1), []string{"col_name", "name2", "name3", "name4"}},
248+ {"SELECT col_name,col_other FROM S3OBJECT AS A LIMIT 5", nil, []string{"col_name", "col_other"}, "A", 5, make([]string, 2), []string{"col_name", "col_other", "name3", "name4"}},
249+ {"SELECT col_name,col_other FROM S3OBJECT AS A WHERE col_name = 'Name' LIMIT 5", nil, []string{"col_name", "col_other"}, "A", 5, make([]string, 2), []string{"col_name", "col_other", "name3", "name4"}},
250+ {"SELECT col_name,col_other FROM S3OBJECT AS A WHERE col_name = 'Name LIMIT 5", ErrLexerInvalidChar, nil, "", 0, nil, []string{"col_name", "col_other", "name3", "name4"}},
251+ {"SELECT count(*) FROM S3OBJECT AS A WHERE col_name = 'Name' LIMIT 5", nil, []string{"*"}, "A", 5, []string{"count"}, []string{"col_name", "col_other", "name3", "name4"}},
252+ {"SELECT sum(col_name),sum(col_other) FROM S3OBJECT AS A WHERE col_name = 'Name' LIMIT 5", nil, []string{"col_name", "col_other"}, "A", 5, []string{"sum", "sum"}, []string{"col_name", "col_other"}},
253+ {"SELECT A.col_name FROM S3OBJECT AS A", nil, []string{"col_name"}, "A", 0, make([]string, 1), []string{"col_name", "col_other", "name3", "name4"}},
254+ {"SELECT A._col_name FROM S3OBJECT AS A", nil, []string{"col_name"}, "A", 0, make([]string, 1), []string{"col_name", "col_other", "name3", "name4"}},
255+ {"SELECT A._col_name FROM S3OBJECT AS A WHERE randomname > 5", ErrMissingHeaders, nil, "", 0, nil, []string{"col_name", "col_other", "name3", "name4"}},
256+ {"SELECT A._col_name FROM S3OBJECT AS A WHERE A._11 > 5", ErrInvalidColumnIndex, nil, "", 0, nil, []string{"col_name", "col_other", "name3", "name4"}},
257+ {"SELECT COALESCE(col_name,col_other) FROM S3OBJECT AS A WHERE A._3 > 5", nil, []string{""}, "A", 0, []string{""}, []string{"col_name", "col_other", "name3", "name4"}},
258+ {"SELECT COALESCE(col_name,col_other),COALESCE(col_name,col_other) FROM S3OBJECT AS A WHERE A._3 > 5", nil, []string{"", ""}, "A", 0, []string{"", ""}, []string{"col_name", "col_other", "name3", "name4"}},
259+ {"SELECT COALESCE(col_name,col_other) ,col_name , COALESCE(col_name,col_other) FROM S3OBJECT AS A WHERE col_name > 5", nil, []string{"", "col_name", ""}, "A", 0, []string{"", "", ""}, []string{"col_name", "col_other", "name3", "name4"}},
260+ {"SELECT NULLIF(col_name,col_other) ,col_name , COALESCE(col_name,col_other) FROM S3OBJECT AS A WHERE col_name > 5", nil, []string{"", "col_name", ""}, "A", 0, []string{"", "", ""}, []string{"col_name", "col_other", "name3", "name4"}},
261+ {"SELECT NULLIF(col_name,col_other) FROM S3OBJECT AS A WHERE col_name > 5", nil, []string{""}, "A", 0, []string{""}, []string{"col_name", "col_other", "name3", "name4"}},
262+ {"SELECT NULLIF(randomname,col_other) FROM S3OBJECT AS A WHERE col_name > 5", ErrMissingHeaders, nil, "", 0, nil, []string{"col_name", "col_other", "name3", "name4"}},
263+ {"SELECT col_name FROM S3OBJECT AS A WHERE COALESCE(random,5) > 5", ErrMissingHeaders, nil, "", 0, nil, []string{"col_name", "col_other", "name3", "name4"}},
264+ {"SELECT col_name FROM S3OBJECT AS A WHERE NULLIF(random,5) > 5", ErrMissingHeaders, nil, "", 0, nil, []string{"col_name", "col_other", "name3", "name4"}},
265+ {"SELECT col_name FROM S3OBJECT AS A WHERE LOWER(col_name) BETWEEN 5 AND 7", nil, []string{"col_name"}, "A", 0, []string{""}, []string{"col_name", "col_other", "name3", "name4"}},
266+ {"SELECT UPPER(col_name) FROM S3OBJECT AS A WHERE LOWER(col_name) BETWEEN 5 AND 7", nil, []string{""}, "A", 0, []string{""}, []string{"col_name", "col_other", "name3", "name4"}},
267+ {"SELECT UPPER(*) FROM S3OBJECT AS A WHERE LOWER(col_name) BETWEEN 5 AND 7", ErrParseUnsupportedCallWithStar, nil, "", 0, nil, []string{"col_name", "col_other", "name3", "name4"}},
268+ {"SELECT NULLIF(col_name,col_name) FROM S3OBJECT AS A WHERE NULLIF(LOWER(col_name),col_name) BETWEEN 5 AND 7", nil, []string{""}, "A", 0, []string{""}, []string{"col_name", "col_other", "name3", "name4"}},
269+ {"SELECT COALESCE(col_name,col_name) FROM S3OBJECT AS A WHERE NULLIF(LOWER(col_name),col_name) BETWEEN 5 AND 7", nil, []string{""}, "A", 0, []string{""}, []string{"col_name", "col_other", "name3", "name4"}},
270+ }
271+ for _, table := range tables {
272+ options := &Options{
273+ HasHeader: false,
274+ FieldDelimiter: ",",
275+ Comments: "",
276+ Name: "S3Object", // Default table name for all objects
277+ ReadFrom: bytes.NewReader([]byte("name1,name2,name3,name4" + "\n" + "5,is,a,string" + "\n" + "random,random,stuff,stuff")),
278+ Compressed: "",
279+ Expression: "",
280+ OutputFieldDelimiter: ",",
281+ StreamSize: 20,
282+ HeaderOpt: true,
283+ }
284+ s3s, err := NewInput(options)
285+ if err != nil {
286+ t.Error(err)
287+ }
288+ s3s.header = table.header
289+ reqCols, alias, myLimit, _, aggFunctionNames, _, err := s3s.ParseSelect(table.myQuery)
290+ if table.err != err {
291+ t.Error()
292+ }
293+ if !reflect.DeepEqual(reqCols, table.reqCols) {
294+ t.Error()
295+ }
296+ if alias != table.alias {
297+ t.Error()
298+ }
299+ if myLimit != table.myLimit {
300+ t.Error()
301+ }
302+ if !reflect.DeepEqual(table.aggFuncs, aggFunctionNames) {
303+ t.Error()
304+ }
305+ }
306+}
307+
308+// Unit tests for the main function that performs aggreggation.
309+func TestMyAggregationFunc(t *testing.T) {
310+ columnsMap := make(map[string]int)
311+ columnsMap["Col1"] = 0
312+ columnsMap["Col2"] = 1
313+ tables := []struct {
314+ counter int
315+ filtrCount int
316+ myAggVals []float64
317+ columnsMap map[string]int
318+ storeReqCols []string
319+ storeFunctions []string
320+ record []string
321+ err error
322+ expectedVal float64
323+ }{
324+ {10, 5, []float64{10}, columnsMap, []string{"Col1"}, []string{"count"}, []string{"1", "2"}, nil, 11},
325+ {10, 5, []float64{10}, columnsMap, []string{"Col1"}, []string{"min"}, []string{"1", "2"}, nil, 1},
326+ {10, 5, []float64{10}, columnsMap, []string{"Col1"}, []string{"max"}, []string{"1", "2"}, nil, 10},
327+ {10, 5, []float64{10}, columnsMap, []string{"Col1"}, []string{"sum"}, []string{"1", "2"}, nil, 11},
328+ {1, 1, []float64{10}, columnsMap, []string{"Col1"}, []string{"avg"}, []string{"1", "2"}, nil, 5.500},
329+ {10, 5, []float64{0.000}, columnsMap, []string{"Col1"}, []string{"random"}, []string{"1", "2"}, ErrParseNonUnaryAgregateFunctionCall, 0},
330+ {0, 5, []float64{0}, columnsMap, []string{"0"}, []string{"count"}, []string{"1", "2"}, nil, 1},
331+ {10, 5, []float64{10}, columnsMap, []string{"1"}, []string{"min"}, []string{"1", "12"}, nil, 10},
332+ }
333+ for _, table := range tables {
334+ err := aggregationFunctions(table.counter, table.filtrCount, table.myAggVals, table.columnsMap, table.storeReqCols, table.storeFunctions, table.record)
335+ if table.err != err {
336+ t.Error()
337+ }
338+ if table.myAggVals[0] != table.expectedVal {
339+ t.Error()
340+ }
341+
342+ }
343+}
344+
345+// Unit Tests for the function which converts a float array to string.
346+func TestToStringAgg(t *testing.T) {
347+ options := &Options{
348+ HasHeader: false,
349+ FieldDelimiter: ",",
350+ Comments: "",
351+ Name: "S3Object", // Default table name for all objects
352+ ReadFrom: bytes.NewReader([]byte("Here , is, a, string + \n + random,random,stuff,stuff ")),
353+ Compressed: "",
354+ Expression: "",
355+ OutputFieldDelimiter: ",",
356+ StreamSize: 20,
357+ HeaderOpt: true,
358+ }
359+ s3s, err := NewInput(options)
360+ if err != nil {
361+ t.Error(err)
362+ }
363+ tables := []struct {
364+ myAggVal []float64
365+ expected string
366+ }{
367+ {[]float64{10, 11, 12, 13, 14}, "10.000000,11.000000,12.000000,13.000000,14.000000"},
368+ {[]float64{10}, "10.000000"},
369+ }
370+ for _, table := range tables {
371+ val := s3s.aggFuncToStr(table.myAggVal)
372+ if val != table.expected {
373+ t.Error()
374+ }
375+ }
376+}
377+
378+// TestMyRowColLiteralResults is a unit test which makes sure that the rows that
379+// are being printed are appropriate to the query being requested.
380+func TestMyRowColLiteralResults(t *testing.T) {
381+ options := &Options{
382+ HasHeader: false,
383+ FieldDelimiter: ",",
384+ Comments: "",
385+ Name: "S3Object", // Default table name for all objects
386+ ReadFrom: bytes.NewReader([]byte("Here , is, a, string + \n + random,random,stuff,stuff ")),
387+ Compressed: "",
388+ Expression: "",
389+ OutputFieldDelimiter: ",",
390+ StreamSize: 20,
391+ HeaderOpt: true,
392+ }
393+ s3s, err := NewInput(options)
394+ if err != nil {
395+ t.Error(err)
396+ }
397+ tables := []struct {
398+ myReq []string
399+ myHeaders map[string]int
400+ myDup map[string]bool
401+ myLow map[string]int
402+ myOpts *Options
403+ tempList []string
404+ input *Input
405+ myRecord []string
406+ myTarget string
407+ columns []string
408+ err error
409+ }{
410+ {[]string{"draft", "year"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, []string{"draft", "year"}, s3s, []string{"target", "random", "hello", "stuff"}, "target,random", []string{"draft", "year", "random", "another"}, nil},
411+ {[]string{"year", "draft"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, []string{"year", "draft"}, s3s, []string{"draft", "2012", "thing", "stuff"}, "2012,draft", []string{"draft", "year", "random", "another"}, nil},
412+ {[]string{"yearrandomstuff", "draft"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, []string{"yearrandomstuff", "draft"}, s3s, []string{"draft", "2012", "thing", "stuff"}, "", []string{"draft", "year", "random", "another"}, ErrMissingHeaders},
413+ {[]string{"draft", "randomstuff"}, make(map[string]int), make(map[string]bool), make(map[string]int), options, []string{"yearrandomstuff", "draft"}, s3s, []string{"draft", "2012", "thing", "stuff"}, "", []string{"draft", "year", "random", "another"}, ErrMissingHeaders},
414+ }
415+ for _, table := range tables {
416+ checkForDuplicates(table.columns, table.myHeaders, table.myDup, table.myLow)
417+ myRow, err := table.input.processColNameLiteral(table.myRecord, table.myReq, table.tempList, table.myHeaders, nil)
418+ if err != table.err {
419+ t.Error()
420+ }
421+ if myRow != table.myTarget {
422+ t.Error()
423+ }
424+ }
425+}
426+
427+// TestMyWhereEval is a function which provides unit tests for the function
428+// which evaluates the where clause.
429+func TestMyWhereEval(t *testing.T) {
430+ columnsMap := make(map[string]int)
431+ columnsMap["Col1"] = 0
432+ columnsMap["Col2"] = 1
433+ tables := []struct {
434+ myQuery string
435+ record []string
436+ err error
437+ expected bool
438+ header []string
439+ }{
440+ {"SELECT * FROM S3OBJECT", []string{"record_1,record_2,record_3,record_4"}, nil, true, []string{"Col1", "Col2"}},
441+ {"SELECT * FROM S3OBJECT WHERE Col1 < -1", []string{"0", "1"}, nil, false, []string{"Col1", "Col2"}},
442+ {"SELECT * FROM S3OBJECT WHERE Col1 < -1 OR Col2 > 15", []string{"151", "12"}, nil, false, []string{"Col1", "Col2"}},
443+ {"SELECT * FROM S3OBJECT WHERE Col1 > -1 AND Col2 > 15", []string{"151", "12"}, nil, false, []string{"Col1", "Col2"}},
444+ {"SELECT * FROM S3OBJECT WHERE Col1 > 1.00", []string{"151.0000", "12"}, nil, true, []string{"Col1", "Col2"}},
445+ {"SELECT * FROM S3OBJECT WHERE Col1 > 100", []string{"random", "12"}, nil, false, []string{"Col1", "Col2"}},
446+ {"SELECT * FROM S3OBJECT WHERE Col1 BETWEEN 100 AND 0", []string{"151", "12"}, nil, false, []string{"Col1", "Col2"}},
447+ {"SELECT * FROM S3OBJECT WHERE Col1 BETWEEN 100.0 AND 0.0", []string{"151", "12"}, nil, false, []string{"Col1", "Col2"}},
448+ {"SELECT * FROM S3OBJECT AS A WHERE A.1 BETWEEN 160 AND 150", []string{"151", "12"}, nil, true, []string{"Col1", "Col2"}},
449+ {"SELECT * FROM S3OBJECT AS A WHERE A._1 BETWEEN 160 AND 0", []string{"151", "12"}, nil, true, []string{"Col1", "Col2"}},
450+ {"SELECT * FROM S3OBJECT AS A WHERE A._1 BETWEEN 0 AND 160", []string{"151", "12"}, nil, true, []string{"Col1", "Col2"}},
451+ {"SELECT * FROM S3OBJECT A._1 LIKE 'r%'", []string{"record_1,record_2,record_3,record_4"}, nil, true, []string{"Col1", "Col2"}},
452+ {"SELECT s._2 FROM S3Object s WHERE s._2 = 'Steven'", []string{"record_1", "Steven", "Steven", "record_4"}, nil, true, []string{"Col1", "Col2"}},
453+ {"SELECT * FROM S3OBJECT AS A WHERE Col1 BETWEEN 0 AND 160", []string{"151", "12"}, nil, true, []string{"Col1", "Col2"}},
454+ {"SELECT * FROM S3OBJECT AS A WHERE Col1 BETWEEN 160 AND 0", []string{"151", "12"}, nil, true, []string{"Col1", "Col2"}},
455+ {"SELECT * FROM S3OBJECT AS A WHERE UPPER(Col1) BETWEEN 160 AND 0", []string{"151", "12"}, nil, true, []string{"Col1", "Col2"}},
456+ {"SELECT * FROM S3OBJECT AS A WHERE UPPER(Col1) = 'RANDOM'", []string{"random", "12"}, nil, true, []string{"Col1", "Col2"}},
457+ {"SELECT * FROM S3OBJECT AS A WHERE LOWER(UPPER(Col1) = 'random'", []string{"random", "12"}, nil, true, []string{"Col1", "Col2"}},
458+ }
459+ for _, table := range tables {
460+ options := &Options{
461+ HasHeader: false,
462+ FieldDelimiter: ",",
463+ Comments: "",
464+ Name: "S3Object", // Default table name for all objects
465+ ReadFrom: bytes.NewReader([]byte("name1,name2,name3,name4" + "\n" + "5,is,a,string" + "\n" + "random,random,stuff,stuff")),
466+ Compressed: "",
467+ Expression: "",
468+ OutputFieldDelimiter: ",",
469+ StreamSize: 20,
470+ HeaderOpt: true,
471+ }
472+ s3s, err := NewInput(options)
473+ s3s.header = table.header
474+
475+ if err != nil {
476+ t.Error(err)
477+ }
478+ _, alias, _, whereClause, _, _, _ := s3s.ParseSelect(table.myQuery)
479+ myVal, err := matchesMyWhereClause(table.record, columnsMap, alias, whereClause)
480+ if table.err != err {
481+ t.Error()
482+ }
483+ if myVal != table.expected {
484+ t.Error()
485+ }
486+ }
487+}
488+
489+// TestMyStringComparator is a unit test which ensures that the appropriate
490+// values are being compared for strings.
491+func TestMyStringComparator(t *testing.T) {
492+ tables := []struct {
493+ operand string
494+ operator string
495+ myVal string
496+ expected bool
497+ err error
498+ }{
499+ {"random", ">", "myName", "random" > "myName", nil},
500+ {"12", "!=", "myName", "12" != "myName", nil},
501+ {"12", "=", "myName", "12" == "myName", nil},
502+ {"12", "<=", "myName", "12" <= "myName", nil},
503+ {"12", ">=", "myName", "12" >= "myName", nil},
504+ {"12", "<", "myName", "12" < "myName", nil},
505+ {"name", "like", "_x%", false, nil},
506+ {"12", "randomoperator", "myName", false, ErrUnsupportedSyntax},
507+ }
508+ for _, table := range tables {
509+ myVal, err := stringEval(table.operand, table.operator, table.myVal)
510+ if err != table.err {
511+ t.Error()
512+ }
513+ if myVal != table.expected {
514+ t.Error()
515+ }
516+ }
517+}
518+
519+// TestMyFloatComparator is a unit test which ensures that the appropriate
520+// values are being compared for floats.
521+func TestMyFloatComparator(t *testing.T) {
522+ tables := []struct {
523+ operand float64
524+ operator string
525+ myVal float64
526+ expected bool
527+ err error
528+ }{
529+ {12.000, ">", 13.000, 12.000 > 13.000, nil},
530+ {1000.000, "!=", 1000.000, 1000.000 != 1000.000, nil},
531+ {1000.000, "<", 1000.000, 1000.000 < 1000.000, nil},
532+ {1000.000, "<=", 1000.000, 1000.000 <= 1000.000, nil},
533+ {1000.000, ">=", 1000.000, 1000.000 >= 1000.000, nil},
534+ {1000.000, "=", 1000.000, 1000.000 == 1000.000, nil},
535+ {17.000, "randomoperator", 0.0, false, ErrUnsupportedSyntax},
536+ }
537+ for _, table := range tables {
538+ myVal, err := floatEval(table.operand, table.operator, table.myVal)
539+ if err != table.err {
540+ t.Error()
541+ }
542+ if myVal != table.expected {
543+ t.Error()
544+ }
545+ }
546+}
547+
548+// TestMyIntComparator is a unit test which ensures that the appropriate values
549+// are being compared for ints.
550+func TestMyIntComparator(t *testing.T) {
551+ tables := []struct {
552+ operand int64
553+ operator string
554+ myVal int64
555+ expected bool
556+ err error
557+ }{
558+ {12, ">", 13, 12.000 > 13.000, nil},
559+ {1000, "!=", 1000, 1000.000 != 1000.000, nil},
560+ {1000, "<", 1000, 1000.000 < 1000.000, nil},
561+ {1000, "<=", 1000, 1000.000 <= 1000.000, nil},
562+ {1000, ">=", 1000, 1000.000 >= 1000.000, nil},
563+ {1000, "=", 1000, 1000.000 >= 1000.000, nil},
564+ {17, "randomoperator", 0, false, ErrUnsupportedSyntax},
565+ }
566+ for _, table := range tables {
567+ myVal, err := intEval(table.operand, table.operator, table.myVal)
568+ if err != table.err {
569+ t.Error()
570+ }
571+ if myVal != table.expected {
572+ t.Error()
573+ }
574+ }
575+}
576+
577+// TestMySizeFunction is a function which provides unit testing for the function
578+// which calculates size.
579+func TestMySizeFunction(t *testing.T) {
580+ tables := []struct {
581+ myRecord []string
582+ expected int64
583+ }{
584+ {[]string{"test1", "test2", "test3", "test4", "test5"}, 30},
585+ }
586+ for _, table := range tables {
587+ if processSize(table.myRecord) != table.expected {
588+ t.Error()
589+ }
590+
591+ }
592+}
593+
594+// TestInterpreter is a function which provides unit testing for the main
595+// interpreter function.
596+func TestInterpreter(t *testing.T) {
597+ tables := []struct {
598+ myQuery string
599+ myChan chan *Row
600+ err error
601+ header []string
602+ }{
603+ {"Select random from S3OBJECT", make(chan *Row), ErrMissingHeaders, []string{"name1", "name2", "name3", "name4"}},
604+ {"Select * from S3OBJECT as A WHERE name2 > 5.00", make(chan *Row), nil, []string{"name1", "name2", "name3", "name4"}},
605+ {"Select * from S3OBJECT", make(chan *Row), nil, []string{"name1", "name2", "name3", "name4"}},
606+ {"Select A_1 from S3OBJECT as A", make(chan *Row), nil, []string{"1", "2", "3", "4"}},
607+ {"Select count(*) from S3OBJECT", make(chan *Row), nil, []string{"name1", "name2", "name3", "name4"}},
608+ {"Select * from S3OBJECT WHERE name1 > 5.00", make(chan *Row), nil, []string{"name1", "name2", "name3", "name4"}},
609+ }
610+ for _, table := range tables {
611+ options := &Options{
612+ HasHeader: false,
613+ FieldDelimiter: ",",
614+ Comments: "",
615+ Name: "S3Object", // Default table name for all objects
616+ ReadFrom: bytes.NewReader([]byte("name1,name2,name3,name4" + "\n" + "5,is,a,string" + "\n" + "random,random,stuff,stuff")),
617+ Compressed: "",
618+ Expression: "",
619+ OutputFieldDelimiter: ",",
620+ StreamSize: 20,
621+ HeaderOpt: true,
622+ }
623+ s3s, err := NewInput(options)
624+ if err != nil {
625+ t.Error(err)
626+ }
627+ s3s.header = table.header
628+ reqCols, alias, myLimit, whereClause, aggFunctionNames, _, err := s3s.ParseSelect(table.myQuery)
629+ if err != table.err {
630+ t.Fatal()
631+ }
632+ if err == nil {
633+ go s3s.processSelectReq(reqCols, alias, whereClause, myLimit, aggFunctionNames, table.myChan, nil)
634+ select {
635+ case row, ok := <-table.myChan:
636+ if ok && len(row.record) > 0 {
637+ } else if ok && row.err != nil {
638+ if row.err != table.err {
639+ t.Error()
640+ }
641+ close(table.myChan)
642+ } else if !ok {
643+ }
644+ }
645+ }
646+ }
647+}
648+
649+// TestMyXMLFunction is a function that provides unit testing for the XML
650+// creating function.
651+func TestMyXMLFunction(t *testing.T) {
652+ options := &Options{
653+ HasHeader: false,
654+ FieldDelimiter: ",",
655+ Comments: "",
656+ Name: "S3Object", // Default table name for all objects
657+ ReadFrom: bytes.NewReader([]byte("name1,name2,name3,name4" + "\n" + "5,is,a,string" + "\n" + "random,random,stuff,stuff")),
658+ Compressed: "",
659+ Expression: "",
660+ OutputFieldDelimiter: ",",
661+ StreamSize: 20,
662+ HeaderOpt: true,
663+ }
664+ s3s, err := NewInput(options)
665+ if err != nil {
666+ t.Error(err)
667+ }
668+ tables := []struct {
669+ expectedStat int
670+ expectedProgress int
671+ }{
672+ {159, 165},
673+ }
674+ for _, table := range tables {
675+ myVal, _ := s3s.createStatXML()
676+ myOtherVal, _ := s3s.createProgressXML()
677+ if len(myVal) != table.expectedStat {
678+ t.Error()
679+ }
680+ if len(myOtherVal) != table.expectedProgress {
681+ fmt.Println(len(myOtherVal))
682+ t.Error()
683+ }
684+ }
685+}
686+
687+// TestMyProtocolFunction is a function which provides unit testing for several
688+// of the functions which write the binary protocol.
689+func TestMyProtocolFunction(t *testing.T) {
690+ options := &Options{
691+ HasHeader: false,
692+ FieldDelimiter: ",",
693+ Comments: "",
694+ Name: "S3Object", // Default table name for all objects
695+ ReadFrom: bytes.NewReader([]byte("name1,name2,name3,name4" + "\n" + "5,is,a,string" + "\n" + "random,random,stuff,stuff")),
696+ Compressed: "",
697+ Expression: "",
698+ OutputFieldDelimiter: ",",
699+ StreamSize: 20,
700+ HeaderOpt: true,
701+ }
702+ s3s, err := NewInput(options)
703+ if err != nil {
704+ t.Error(err)
705+ }
706+ tables := []struct {
707+ payloadMsg string
708+ expectedRecord int
709+ expectedEnd int
710+ }{
711+ {"random payload", 115, 56},
712+ }
713+ for _, table := range tables {
714+ var currentMessage = &bytes.Buffer{}
715+ if len(s3s.writeRecordMessage(table.payloadMsg, currentMessage).Bytes()) != table.expectedRecord {
716+ t.Error()
717+ }
718+ currentMessage.Reset()
719+ if len(s3s.writeEndMessage(currentMessage).Bytes()) != table.expectedEnd {
720+ t.Error()
721+ }
722+ currentMessage.Reset()
723+ if len(s3s.writeContinuationMessage(currentMessage).Bytes()) != 57 {
724+ t.Error()
725+ }
726+ currentMessage.Reset()
727+ }
728+}
729+
730+// TestMyInfoProtocolFunctions is a function which provides unit testing for the
731+// stat and progress messages of the protocols.
732+func TestMyInfoProtocolFunctions(t *testing.T) {
733+ options := &Options{
734+ HasHeader: true,
735+ FieldDelimiter: ",",
736+ Comments: "",
737+ Name: "S3Object", // Default table name for all objects
738+ ReadFrom: bytes.NewReader([]byte("name1,name2,name3,name4" + "\n" + "5,is,a,string" + "\n" + "random,random,stuff,stuff")),
739+ Compressed: "",
740+ Expression: "",
741+ OutputFieldDelimiter: ",",
742+ StreamSize: 20,
743+ }
744+ s3s, err := NewInput(options)
745+ if err != nil {
746+ t.Error(err)
747+ }
748+ myVal, _ := s3s.createStatXML()
749+ myOtherVal, _ := s3s.createProgressXML()
750+
751+ tables := []struct {
752+ payloadStatMsg string
753+ payloadProgressMsg string
754+ expectedStat int
755+ expectedProgress int
756+ }{
757+ {myVal, myOtherVal, 242, 252},
758+ }
759+ for _, table := range tables {
760+ var currBuf = &bytes.Buffer{}
761+ if len(s3s.writeStatMessage(table.payloadStatMsg, currBuf).Bytes()) != table.expectedStat {
762+ t.Error()
763+ }
764+ currBuf.Reset()
765+ if len(s3s.writeProgressMessage(table.payloadProgressMsg, currBuf).Bytes()) != table.expectedProgress {
766+ t.Error()
767+ }
768+ }
769+}
770+
771+// TestMyErrorProtocolFunctions is a function which provides unit testing for
772+// the error message type of protocol.
773+func TestMyErrorProtocolFunctions(t *testing.T) {
774+ options := &Options{
775+ HasHeader: false,
776+ FieldDelimiter: ",",
777+ Comments: "",
778+ Name: "S3Object", // Default table name for all objects
779+ ReadFrom: bytes.NewReader([]byte("name1,name2,name3,name4" + "\n" + "5,is,a,string" + "\n" + "random,random,stuff,stuff")),
780+ Compressed: "",
781+ Expression: "",
782+ OutputFieldDelimiter: ",",
783+ StreamSize: 20,
784+ HeaderOpt: true,
785+ }
786+ s3s, err := NewInput(options)
787+ if err != nil {
788+ t.Error(err)
789+ }
790+ tables := []struct {
791+ err error
792+ expectedError int
793+ }{
794+ {ErrInvalidCast, 248},
795+ {ErrTruncatedInput, 200},
796+ {ErrUnsupportedSyntax, 114},
797+ {ErrCSVParsingError, 157},
798+ }
799+ for _, table := range tables {
800+ var currentMessage = &bytes.Buffer{}
801+ if len(s3s.writeErrorMessage(table.err, currentMessage).Bytes()) != table.expectedError {
802+ t.Error()
803+ }
804+
805+ }
806+}
807+func TestMatch(t *testing.T) {
808+ testCases := []struct {
809+ pattern string
810+ text string
811+ matched bool
812+ }{
813+ // Test case - 1.
814+ // Test case so that the match occurs on the opening letter.
815+ {
816+ pattern: "a%",
817+ text: "apple",
818+ matched: true,
819+ },
820+ // Test case - 2.
821+ // Test case so that the ending letter is true.
822+ {
823+ pattern: "%m",
824+ text: "random",
825+ matched: true,
826+ },
827+ // Test case - 3.
828+ // Test case so that a character is at the appropriate position.
829+ {
830+ pattern: "_d%",
831+ text: "adam",
832+ matched: true,
833+ },
834+ // Test case - 4.
835+ // Test case so that a character is at the appropriate position.
836+ {
837+ pattern: "_d%",
838+ text: "apple",
839+ matched: false,
840+ },
841+ // Test case - 5.
842+ // Test case with checking that it is at least 3 in length
843+ {
844+ pattern: "a_%_%",
845+ text: "ap",
846+ matched: false,
847+ },
848+ {
849+ pattern: "a_%_%",
850+ text: "apple",
851+ matched: true,
852+ },
853+ {
854+ pattern: "%or%",
855+ text: "orphan",
856+ matched: true,
857+ },
858+ {
859+ pattern: "%or%",
860+ text: "dolphin",
861+ matched: false,
862+ },
863+ {
864+ pattern: "%or%",
865+ text: "dorlphin",
866+ matched: true,
867+ },
868+ {
869+ pattern: "2__3",
870+ text: "2003",
871+ matched: true,
872+ },
873+ {
874+ pattern: "_YYYY_",
875+ text: "aYYYYa",
876+ matched: true,
877+ },
878+ {
879+ pattern: "C%",
880+ text: "CA",
881+ matched: true,
882+ },
883+ {
884+ pattern: "C%",
885+ text: "SC",
886+ matched: false,
887+ },
888+ {
889+ pattern: "%C",
890+ text: "SC",
891+ matched: true,
892+ },
893+ {
894+ pattern: "%C",
895+ text: "CA",
896+ matched: false,
897+ },
898+ {
899+ pattern: "%C",
900+ text: "ACCC",
901+ matched: true,
902+ },
903+ {
904+ pattern: "C%",
905+ text: "CCC",
906+ matched: true,
907+ },
908+ {
909+ pattern: "j%",
910+ text: "mejri",
911+ matched: false,
912+ },
913+ {
914+ pattern: "a%o",
915+ text: "ando",
916+ matched: true,
917+ },
918+ {
919+ pattern: "%j",
920+ text: "mejri",
921+ matched: false,
922+ },
923+ {
924+ pattern: "%ja",
925+ text: "mejrija",
926+ matched: true,
927+ },
928+ {
929+ pattern: "ja%",
930+ text: "jamal",
931+ matched: true,
932+ },
933+ {
934+ pattern: "a%o",
935+ text: "andp",
936+ matched: false,
937+ },
938+ {
939+ pattern: "_r%",
940+ text: "arpa",
941+ matched: true,
942+ },
943+ {
944+ pattern: "_r%",
945+ text: "apra",
946+ matched: false,
947+ },
948+ {
949+ pattern: "a_%_%",
950+ text: "appple",
951+ matched: true,
952+ },
953+ {
954+ pattern: "l_b%",
955+ text: "lebron",
956+ matched: true,
957+ },
958+ {
959+ pattern: "leb%",
960+ text: "Dalembert",
961+ matched: false,
962+ },
963+ {
964+ pattern: "leb%",
965+ text: "Landesberg",
966+ matched: false,
967+ },
968+ {
969+ pattern: "leb%",
970+ text: "Mccalebb",
971+ matched: false,
972+ },
973+ {
974+ pattern: "%lebb",
975+ text: "Mccalebb",
976+ matched: true,
977+ },
978+ }
979+ // Iterating over the test cases, call the function under test and asert the output.
980+ for i, testCase := range testCases {
981+ actualResult, err := likeConvert(testCase.pattern, testCase.text)
982+ if err != nil {
983+ t.Error()
984+ }
985+ if testCase.matched != actualResult {
986+ fmt.Println("Expected Pattern", testCase.pattern, "Expected Text", testCase.text)
987+ t.Errorf("Test %d: Expected the result to be `%v`, but instead found it to be `%v`", i+1, testCase.matched, actualResult)
988+ }
989+ }
990+}
991+
992+// TestMyValids is a unit test which ensures that the appropriate values are
993+// being returned from the isValid... functions.
994+func TestMyValids(t *testing.T) {
995+
996+ tables := []struct {
997+ myQuery string
998+ indexList []int
999+ myIndex int
1000+ myValIndex bool
1001+ header []string
1002+ err error
1003+ }{
1004+ {"SELECT UPPER(NULLIF(draft_year,random_name))", []int{3, 5, 6, 7, 8, 9}, 3, true, []string{"draft_year", "random_name"}, nil},
1005+ {"SELECT UPPER(NULLIF(draft_year,xandom_name))", []int{3, 5, 6, 7, 8, 9}, 3, true, []string{"draft_year", "random_name"}, ErrMissingHeaders},
1006+ }
1007+ for _, table := range tables {
1008+ options := &Options{
1009+ HasHeader: false,
1010+ FieldDelimiter: ",",
1011+ Comments: "",
1012+ Name: "S3Object", // Default table name for all objects
1013+ ReadFrom: bytes.NewReader([]byte("name1,name2,name3,name4" + "\n" + "5,is,a,string" + "\n" + "random,random,stuff,stuff")),
1014+ Compressed: "",
1015+ Expression: "",
1016+ OutputFieldDelimiter: ",",
1017+ StreamSize: 20,
1018+ HeaderOpt: true,
1019+ }
1020+ s3s, err := NewInput(options)
1021+ if err != nil {
1022+ t.Error(err)
1023+ }
1024+ s3s.header = table.header
1025+ _, _, _, _, _, _, err = s3s.ParseSelect(table.myQuery)
1026+ if err != table.err {
1027+ t.Fatal()
1028+ }
1029+ myVal := isValidFunc(table.indexList, table.myIndex)
1030+ if myVal != table.myValIndex {
1031+ t.Error()
1032+ }
1033+ }
1034+}
1035+
1036+// TestMyFuncProcessing is a unit test which ensures that the appropriate values are
1037+// being returned from the Processing... functions.
1038+func TestMyFuncProcessing(t *testing.T) {
1039+ tables := []struct {
1040+ myString string
1041+ nullList []string
1042+ coalList []string
1043+ myValString string
1044+ myValCoal string
1045+ myValNull string
1046+ stringFunc string
1047+ }{
1048+ {"lower", []string{"yo", "yo"}, []string{"random", "hello", "random"}, "LOWER", "random", "", "UPPER"},
1049+ {"LOWER", []string{"null", "random"}, []string{"missing", "hello", "random"}, "lower", "hello", "null", "LOWER"},
1050+ }
1051+ for _, table := range tables {
1052+ if table.coalList != nil {
1053+ myVal := processCoalNoIndex(table.coalList)
1054+ if myVal != table.myValCoal {
1055+ t.Error()
1056+ }
1057+ }
1058+ if table.nullList != nil {
1059+ myVal := processNullIf(table.nullList)
1060+ if myVal != table.myValNull {
1061+ t.Error()
1062+ }
1063+ }
1064+ myVal := applyStrFunc(table.myString, table.stringFunc)
1065+ if myVal != table.myValString {
1066+ t.Error()
1067+ }
1068+
1069+ }
1070+}
vendor/github.com/klauspost/compress/LICENSE+27−0
@@ -0,0 +1,27 @@
1+Copyright (c) 2012 The Go Authors. All rights reserved.
2+
3+Redistribution and use in source and binary forms, with or without
4+modification, are permitted provided that the following conditions are
5+met:
6+
7+ * Redistributions of source code must retain the above copyright
8+notice, this list of conditions and the following disclaimer.
9+ * Redistributions in binary form must reproduce the above
10+copyright notice, this list of conditions and the following disclaimer
11+in the documentation and/or other materials provided with the
12+distribution.
13+ * Neither the name of Google Inc. nor the names of its
14+contributors may be used to endorse or promote products derived from
15+this software without specific prior written permission.
16+
17+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
vendor/github.com/klauspost/compress/flate/copy.go+32−0
@@ -0,0 +1,32 @@
1+// Copyright 2012 The Go Authors. All rights reserved.
2+// Use of this source code is governed by a BSD-style
3+// license that can be found in the LICENSE file.
4+
5+package flate
6+
7+// forwardCopy is like the built-in copy function except that it always goes
8+// forward from the start, even if the dst and src overlap.
9+// It is equivalent to:
10+// for i := 0; i < n; i++ {
11+// mem[dst+i] = mem[src+i]
12+// }
13+func forwardCopy(mem []byte, dst, src, n int) {
14+ if dst <= src {
15+ copy(mem[dst:dst+n], mem[src:src+n])
16+ return
17+ }
18+ for {
19+ if dst >= src+n {
20+ copy(mem[dst:dst+n], mem[src:src+n])
21+ return
22+ }
23+ // There is some forward overlap. The destination
24+ // will be filled with a repeated pattern of mem[src:src+k].
25+ // We copy one instance of the pattern here, then repeat.
26+ // Each time around this loop k will double.
27+ k := dst - src
28+ copy(mem[dst:dst+k], mem[src:src+k])
29+ n -= k
30+ dst += k
31+ }
32+}
vendor/github.com/klauspost/compress/flate/crc32_amd64.go+41−0
@@ -0,0 +1,41 @@
1+//+build !noasm
2+//+build !appengine
3+
4+// Copyright 2015, Klaus Post, see LICENSE for details.
5+
6+package flate
7+
8+import (
9+ "github.com/klauspost/cpuid"
10+)
11+
12+// crc32sse returns a hash for the first 4 bytes of the slice
13+// len(a) must be >= 4.
14+//go:noescape
15+func crc32sse(a []byte) uint32
16+
17+// crc32sseAll calculates hashes for each 4-byte set in a.
18+// dst must be east len(a) - 4 in size.
19+// The size is not checked by the assembly.
20+//go:noescape
21+func crc32sseAll(a []byte, dst []uint32)
22+
23+// matchLenSSE4 returns the number of matching bytes in a and b
24+// up to length 'max'. Both slices must be at least 'max'
25+// bytes in size.
26+//
27+// TODO: drop the "SSE4" name, since it doesn't use any SSE instructions.
28+//
29+//go:noescape
30+func matchLenSSE4(a, b []byte, max int) int
31+
32+// histogram accumulates a histogram of b in h.
33+// h must be at least 256 entries in length,
34+// and must be cleared before calling this function.
35+//go:noescape
36+func histogram(b []byte, h []int32)
37+
38+// Detect SSE 4.2 feature.
39+func init() {
40+ useSSE42 = cpuid.CPU.SSE42()
41+}
vendor/github.com/klauspost/compress/flate/crc32_amd64.s+213−0
@@ -0,0 +1,213 @@
1+//+build !noasm
2+//+build !appengine
3+
4+// Copyright 2015, Klaus Post, see LICENSE for details.
5+
6+// func crc32sse(a []byte) uint32
7+TEXT ·crc32sse(SB), 4, $0
8+ MOVQ a+0(FP), R10
9+ XORQ BX, BX
10+
11+ // CRC32 dword (R10), EBX
12+ BYTE $0xF2; BYTE $0x41; BYTE $0x0f
13+ BYTE $0x38; BYTE $0xf1; BYTE $0x1a
14+
15+ MOVL BX, ret+24(FP)
16+ RET
17+
18+// func crc32sseAll(a []byte, dst []uint32)
19+TEXT ·crc32sseAll(SB), 4, $0
20+ MOVQ a+0(FP), R8 // R8: src
21+ MOVQ a_len+8(FP), R10 // input length
22+ MOVQ dst+24(FP), R9 // R9: dst
23+ SUBQ $4, R10
24+ JS end
25+ JZ one_crc
26+ MOVQ R10, R13
27+ SHRQ $2, R10 // len/4
28+ ANDQ $3, R13 // len&3
29+ XORQ BX, BX
30+ ADDQ $1, R13
31+ TESTQ R10, R10
32+ JZ rem_loop
33+
34+crc_loop:
35+ MOVQ (R8), R11
36+ XORQ BX, BX
37+ XORQ DX, DX
38+ XORQ DI, DI
39+ MOVQ R11, R12
40+ SHRQ $8, R11
41+ MOVQ R12, AX
42+ MOVQ R11, CX
43+ SHRQ $16, R12
44+ SHRQ $16, R11
45+ MOVQ R12, SI
46+
47+ // CRC32 EAX, EBX
48+ BYTE $0xF2; BYTE $0x0f
49+ BYTE $0x38; BYTE $0xf1; BYTE $0xd8
50+
51+ // CRC32 ECX, EDX
52+ BYTE $0xF2; BYTE $0x0f
53+ BYTE $0x38; BYTE $0xf1; BYTE $0xd1
54+
55+ // CRC32 ESI, EDI
56+ BYTE $0xF2; BYTE $0x0f
57+ BYTE $0x38; BYTE $0xf1; BYTE $0xfe
58+ MOVL BX, (R9)
59+ MOVL DX, 4(R9)
60+ MOVL DI, 8(R9)
61+
62+ XORQ BX, BX
63+ MOVL R11, AX
64+
65+ // CRC32 EAX, EBX
66+ BYTE $0xF2; BYTE $0x0f
67+ BYTE $0x38; BYTE $0xf1; BYTE $0xd8
68+ MOVL BX, 12(R9)
69+
70+ ADDQ $16, R9
71+ ADDQ $4, R8
72+ XORQ BX, BX
73+ SUBQ $1, R10
74+ JNZ crc_loop
75+
76+rem_loop:
77+ MOVL (R8), AX
78+
79+ // CRC32 EAX, EBX
80+ BYTE $0xF2; BYTE $0x0f
81+ BYTE $0x38; BYTE $0xf1; BYTE $0xd8
82+
83+ MOVL BX, (R9)
84+ ADDQ $4, R9
85+ ADDQ $1, R8
86+ XORQ BX, BX
87+ SUBQ $1, R13
88+ JNZ rem_loop
89+
90+end:
91+ RET
92+
93+one_crc:
94+ MOVQ $1, R13
95+ XORQ BX, BX
96+ JMP rem_loop
97+
98+// func matchLenSSE4(a, b []byte, max int) int
99+TEXT ·matchLenSSE4(SB), 4, $0
100+ MOVQ a_base+0(FP), SI
101+ MOVQ b_base+24(FP), DI
102+ MOVQ DI, DX
103+ MOVQ max+48(FP), CX
104+
105+cmp8:
106+ // As long as we are 8 or more bytes before the end of max, we can load and
107+ // compare 8 bytes at a time. If those 8 bytes are equal, repeat.
108+ CMPQ CX, $8
109+ JLT cmp1
110+ MOVQ (SI), AX
111+ MOVQ (DI), BX
112+ CMPQ AX, BX
113+ JNE bsf
114+ ADDQ $8, SI
115+ ADDQ $8, DI
116+ SUBQ $8, CX
117+ JMP cmp8
118+
119+bsf:
120+ // If those 8 bytes were not equal, XOR the two 8 byte values, and return
121+ // the index of the first byte that differs. The BSF instruction finds the
122+ // least significant 1 bit, the amd64 architecture is little-endian, and
123+ // the shift by 3 converts a bit index to a byte index.
124+ XORQ AX, BX
125+ BSFQ BX, BX
126+ SHRQ $3, BX
127+ ADDQ BX, DI
128+
129+ // Subtract off &b[0] to convert from &b[ret] to ret, and return.
130+ SUBQ DX, DI
131+ MOVQ DI, ret+56(FP)
132+ RET
133+
134+cmp1:
135+ // In the slices' tail, compare 1 byte at a time.
136+ CMPQ CX, $0
137+ JEQ matchLenEnd
138+ MOVB (SI), AX
139+ MOVB (DI), BX
140+ CMPB AX, BX
141+ JNE matchLenEnd
142+ ADDQ $1, SI
143+ ADDQ $1, DI
144+ SUBQ $1, CX
145+ JMP cmp1
146+
147+matchLenEnd:
148+ // Subtract off &b[0] to convert from &b[ret] to ret, and return.
149+ SUBQ DX, DI
150+ MOVQ DI, ret+56(FP)
151+ RET
152+
153+// func histogram(b []byte, h []int32)
154+TEXT ·histogram(SB), 4, $0
155+ MOVQ b+0(FP), SI // SI: &b
156+ MOVQ b_len+8(FP), R9 // R9: len(b)
157+ MOVQ h+24(FP), DI // DI: Histogram
158+ MOVQ R9, R8
159+ SHRQ $3, R8
160+ JZ hist1
161+ XORQ R11, R11
162+
163+loop_hist8:
164+ MOVQ (SI), R10
165+
166+ MOVB R10, R11
167+ INCL (DI)(R11*4)
168+ SHRQ $8, R10
169+
170+ MOVB R10, R11
171+ INCL (DI)(R11*4)
172+ SHRQ $8, R10
173+
174+ MOVB R10, R11
175+ INCL (DI)(R11*4)
176+ SHRQ $8, R10
177+
178+ MOVB R10, R11
179+ INCL (DI)(R11*4)
180+ SHRQ $8, R10
181+
182+ MOVB R10, R11
183+ INCL (DI)(R11*4)
184+ SHRQ $8, R10
185+
186+ MOVB R10, R11
187+ INCL (DI)(R11*4)
188+ SHRQ $8, R10
189+
190+ MOVB R10, R11
191+ INCL (DI)(R11*4)
192+ SHRQ $8, R10
193+
194+ INCL (DI)(R10*4)
195+
196+ ADDQ $8, SI
197+ DECQ R8
198+ JNZ loop_hist8
199+
200+hist1:
201+ ANDQ $7, R9
202+ JZ end_hist
203+ XORQ R10, R10
204+
205+loop_hist1:
206+ MOVB (SI), R10
207+ INCL (DI)(R10*4)
208+ INCQ SI
209+ DECQ R9
210+ JNZ loop_hist1
211+
212+end_hist:
213+ RET
vendor/github.com/klauspost/compress/flate/crc32_noasm.go+35−0
@@ -0,0 +1,35 @@
1+//+build !amd64 noasm appengine
2+
3+// Copyright 2015, Klaus Post, see LICENSE for details.
4+
5+package flate
6+
7+func init() {
8+ useSSE42 = false
9+}
10+
11+// crc32sse should never be called.
12+func crc32sse(a []byte) uint32 {
13+ panic("no assembler")
14+}
15+
16+// crc32sseAll should never be called.
17+func crc32sseAll(a []byte, dst []uint32) {
18+ panic("no assembler")
19+}
20+
21+// matchLenSSE4 should never be called.
22+func matchLenSSE4(a, b []byte, max int) int {
23+ panic("no assembler")
24+ return 0
25+}
26+
27+// histogram accumulates a histogram of b in h.
28+//
29+// len(h) must be >= 256, and h's elements must be all zeroes.
30+func histogram(b []byte, h []int32) {
31+ h = h[:256]
32+ for _, t := range b {
33+ h[t]++
34+ }
35+}
vendor/github.com/klauspost/compress/flate/deflate.go+1353−0
@@ -0,0 +1,1353 @@
1+// Copyright 2009 The Go Authors. All rights reserved.
2+// Copyright (c) 2015 Klaus Post
3+// Use of this source code is governed by a BSD-style
4+// license that can be found in the LICENSE file.
5+
6+package flate
7+
8+import (
9+ "fmt"
10+ "io"
11+ "math"
12+)
13+
14+const (
15+ NoCompression = 0
16+ BestSpeed = 1
17+ BestCompression = 9
18+ DefaultCompression = -1
19+
20+ // HuffmanOnly disables Lempel-Ziv match searching and only performs Huffman
21+ // entropy encoding. This mode is useful in compressing data that has
22+ // already been compressed with an LZ style algorithm (e.g. Snappy or LZ4)
23+ // that lacks an entropy encoder. Compression gains are achieved when
24+ // certain bytes in the input stream occur more frequently than others.
25+ //
26+ // Note that HuffmanOnly produces a compressed output that is
27+ // RFC 1951 compliant. That is, any valid DEFLATE decompressor will
28+ // continue to be able to decompress this output.
29+ HuffmanOnly = -2
30+ ConstantCompression = HuffmanOnly // compatibility alias.
31+
32+ logWindowSize = 15
33+ windowSize = 1 << logWindowSize
34+ windowMask = windowSize - 1
35+ logMaxOffsetSize = 15 // Standard DEFLATE
36+ minMatchLength = 4 // The smallest match that the compressor looks for
37+ maxMatchLength = 258 // The longest match for the compressor
38+ minOffsetSize = 1 // The shortest offset that makes any sense
39+
40+ // The maximum number of tokens we put into a single flat block, just too
41+ // stop things from getting too large.
42+ maxFlateBlockTokens = 1 << 14
43+ maxStoreBlockSize = 65535
44+ hashBits = 17 // After 17 performance degrades
45+ hashSize = 1 << hashBits
46+ hashMask = (1 << hashBits) - 1
47+ hashShift = (hashBits + minMatchLength - 1) / minMatchLength
48+ maxHashOffset = 1 << 24
49+
50+ skipNever = math.MaxInt32
51+)
52+
53+var useSSE42 bool
54+
55+type compressionLevel struct {
56+ good, lazy, nice, chain, fastSkipHashing, level int
57+}
58+
59+// Compression levels have been rebalanced from zlib deflate defaults
60+// to give a bigger spread in speed and compression.
61+// See https://blog.klauspost.com/rebalancing-deflate-compression-levels/
62+var levels = []compressionLevel{
63+ {}, // 0
64+ // Level 1-4 uses specialized algorithm - values not used
65+ {0, 0, 0, 0, 0, 1},
66+ {0, 0, 0, 0, 0, 2},
67+ {0, 0, 0, 0, 0, 3},
68+ {0, 0, 0, 0, 0, 4},
69+ // For levels 5-6 we don't bother trying with lazy matches.
70+ // Lazy matching is at least 30% slower, with 1.5% increase.
71+ {6, 0, 12, 8, 12, 5},
72+ {8, 0, 24, 16, 16, 6},
73+ // Levels 7-9 use increasingly more lazy matching
74+ // and increasingly stringent conditions for "good enough".
75+ {8, 8, 24, 16, skipNever, 7},
76+ {10, 16, 24, 64, skipNever, 8},
77+ {32, 258, 258, 4096, skipNever, 9},
78+}
79+
80+type compressor struct {
81+ compressionLevel
82+
83+ w *huffmanBitWriter
84+ bulkHasher func([]byte, []uint32)
85+
86+ // compression algorithm
87+ fill func(*compressor, []byte) int // copy data to window
88+ step func(*compressor) // process window
89+ sync bool // requesting flush
90+
91+ // Input hash chains
92+ // hashHead[hashValue] contains the largest inputIndex with the specified hash value
93+ // If hashHead[hashValue] is within the current window, then
94+ // hashPrev[hashHead[hashValue] & windowMask] contains the previous index
95+ // with the same hash value.
96+ chainHead int
97+ hashHead [hashSize]uint32
98+ hashPrev [windowSize]uint32
99+ hashOffset int
100+
101+ // input window: unprocessed data is window[index:windowEnd]
102+ index int
103+ window []byte
104+ windowEnd int
105+ blockStart int // window index where current tokens start
106+ byteAvailable bool // if true, still need to process window[index-1].
107+
108+ // queued output tokens
109+ tokens tokens
110+
111+ // deflate state
112+ length int
113+ offset int
114+ hash uint32
115+ maxInsertIndex int
116+ err error
117+ ii uint16 // position of last match, intended to overflow to reset.
118+
119+ snap snappyEnc
120+ hashMatch [maxMatchLength + minMatchLength]uint32
121+}
122+
123+func (d *compressor) fillDeflate(b []byte) int {
124+ if d.index >= 2*windowSize-(minMatchLength+maxMatchLength) {
125+ // shift the window by windowSize
126+ copy(d.window[:], d.window[windowSize:2*windowSize])
127+ d.index -= windowSize
128+ d.windowEnd -= windowSize
129+ if d.blockStart >= windowSize {
130+ d.blockStart -= windowSize
131+ } else {
132+ d.blockStart = math.MaxInt32
133+ }
134+ d.hashOffset += windowSize
135+ if d.hashOffset > maxHashOffset {
136+ delta := d.hashOffset - 1
137+ d.hashOffset -= delta
138+ d.chainHead -= delta
139+ // Iterate over slices instead of arrays to avoid copying
140+ // the entire table onto the stack (Issue #18625).
141+ for i, v := range d.hashPrev[:] {
142+ if int(v) > delta {
143+ d.hashPrev[i] = uint32(int(v) - delta)
144+ } else {
145+ d.hashPrev[i] = 0
146+ }
147+ }
148+ for i, v := range d.hashHead[:] {
149+ if int(v) > delta {
150+ d.hashHead[i] = uint32(int(v) - delta)
151+ } else {
152+ d.hashHead[i] = 0
153+ }
154+ }
155+ }
156+ }
157+ n := copy(d.window[d.windowEnd:], b)
158+ d.windowEnd += n
159+ return n
160+}
161+
162+func (d *compressor) writeBlock(tok tokens, index int, eof bool) error {
163+ if index > 0 || eof {
164+ var window []byte
165+ if d.blockStart <= index {
166+ window = d.window[d.blockStart:index]
167+ }
168+ d.blockStart = index
169+ d.w.writeBlock(tok.tokens[:tok.n], eof, window)
170+ return d.w.err
171+ }
172+ return nil
173+}
174+
175+// writeBlockSkip writes the current block and uses the number of tokens
176+// to determine if the block should be stored on no matches, or
177+// only huffman encoded.
178+func (d *compressor) writeBlockSkip(tok tokens, index int, eof bool) error {
179+ if index > 0 || eof {
180+ if d.blockStart <= index {
181+ window := d.window[d.blockStart:index]
182+ // If we removed less than a 64th of all literals
183+ // we huffman compress the block.
184+ if int(tok.n) > len(window)-int(tok.n>>6) {
185+ d.w.writeBlockHuff(eof, window)
186+ } else {
187+ // Write a dynamic huffman block.
188+ d.w.writeBlockDynamic(tok.tokens[:tok.n], eof, window)
189+ }
190+ } else {
191+ d.w.writeBlock(tok.tokens[:tok.n], eof, nil)
192+ }
193+ d.blockStart = index
194+ return d.w.err
195+ }
196+ return nil
197+}
198+
199+// fillWindow will fill the current window with the supplied
200+// dictionary and calculate all hashes.
201+// This is much faster than doing a full encode.
202+// Should only be used after a start/reset.
203+func (d *compressor) fillWindow(b []byte) {
204+ // Do not fill window if we are in store-only mode,
205+ // use constant or Snappy compression.
206+ switch d.compressionLevel.level {
207+ case 0, 1, 2:
208+ return
209+ }
210+ // If we are given too much, cut it.
211+ if len(b) > windowSize {
212+ b = b[len(b)-windowSize:]
213+ }
214+ // Add all to window.
215+ n := copy(d.window[d.windowEnd:], b)
216+
217+ // Calculate 256 hashes at the time (more L1 cache hits)
218+ loops := (n + 256 - minMatchLength) / 256
219+ for j := 0; j < loops; j++ {
220+ startindex := j * 256
221+ end := startindex + 256 + minMatchLength - 1
222+ if end > n {
223+ end = n
224+ }
225+ tocheck := d.window[startindex:end]
226+ dstSize := len(tocheck) - minMatchLength + 1
227+
228+ if dstSize <= 0 {
229+ continue
230+ }
231+
232+ dst := d.hashMatch[:dstSize]
233+ d.bulkHasher(tocheck, dst)
234+ var newH uint32
235+ for i, val := range dst {
236+ di := i + startindex
237+ newH = val & hashMask
238+ // Get previous value with the same hash.
239+ // Our chain should point to the previous value.
240+ d.hashPrev[di&windowMask] = d.hashHead[newH]
241+ // Set the head of the hash chain to us.
242+ d.hashHead[newH] = uint32(di + d.hashOffset)
243+ }
244+ d.hash = newH
245+ }
246+ // Update window information.
247+ d.windowEnd += n
248+ d.index = n
249+}
250+
251+// Try to find a match starting at index whose length is greater than prevSize.
252+// We only look at chainCount possibilities before giving up.
253+// pos = d.index, prevHead = d.chainHead-d.hashOffset, prevLength=minMatchLength-1, lookahead
254+func (d *compressor) findMatch(pos int, prevHead int, prevLength int, lookahead int) (length, offset int, ok bool) {
255+ minMatchLook := maxMatchLength
256+ if lookahead < minMatchLook {
257+ minMatchLook = lookahead
258+ }
259+
260+ win := d.window[0 : pos+minMatchLook]
261+
262+ // We quit when we get a match that's at least nice long
263+ nice := len(win) - pos
264+ if d.nice < nice {
265+ nice = d.nice
266+ }
267+
268+ // If we've got a match that's good enough, only look in 1/4 the chain.
269+ tries := d.chain
270+ length = prevLength
271+ if length >= d.good {
272+ tries >>= 2
273+ }
274+
275+ wEnd := win[pos+length]
276+ wPos := win[pos:]
277+ minIndex := pos - windowSize
278+
279+ for i := prevHead; tries > 0; tries-- {
280+ if wEnd == win[i+length] {
281+ n := matchLen(win[i:], wPos, minMatchLook)
282+
283+ if n > length && (n > minMatchLength || pos-i <= 4096) {
284+ length = n
285+ offset = pos - i
286+ ok = true
287+ if n >= nice {
288+ // The match is good enough that we don't try to find a better one.
289+ break
290+ }
291+ wEnd = win[pos+n]
292+ }
293+ }
294+ if i == minIndex {
295+ // hashPrev[i & windowMask] has already been overwritten, so stop now.
296+ break
297+ }
298+ i = int(d.hashPrev[i&windowMask]) - d.hashOffset
299+ if i < minIndex || i < 0 {
300+ break
301+ }
302+ }
303+ return
304+}
305+
306+// Try to find a match starting at index whose length is greater than prevSize.
307+// We only look at chainCount possibilities before giving up.
308+// pos = d.index, prevHead = d.chainHead-d.hashOffset, prevLength=minMatchLength-1, lookahead
309+func (d *compressor) findMatchSSE(pos int, prevHead int, prevLength int, lookahead int) (length, offset int, ok bool) {
310+ minMatchLook := maxMatchLength
311+ if lookahead < minMatchLook {
312+ minMatchLook = lookahead
313+ }
314+
315+ win := d.window[0 : pos+minMatchLook]
316+
317+ // We quit when we get a match that's at least nice long
318+ nice := len(win) - pos
319+ if d.nice < nice {
320+ nice = d.nice
321+ }
322+
323+ // If we've got a match that's good enough, only look in 1/4 the chain.
324+ tries := d.chain
325+ length = prevLength
326+ if length >= d.good {
327+ tries >>= 2
328+ }
329+
330+ wEnd := win[pos+length]
331+ wPos := win[pos:]
332+ minIndex := pos - windowSize
333+
334+ for i := prevHead; tries > 0; tries-- {
335+ if wEnd == win[i+length] {
336+ n := matchLenSSE4(win[i:], wPos, minMatchLook)
337+
338+ if n > length && (n > minMatchLength || pos-i <= 4096) {
339+ length = n
340+ offset = pos - i
341+ ok = true
342+ if n >= nice {
343+ // The match is good enough that we don't try to find a better one.
344+ break
345+ }
346+ wEnd = win[pos+n]
347+ }
348+ }
349+ if i == minIndex {
350+ // hashPrev[i & windowMask] has already been overwritten, so stop now.
351+ break
352+ }
353+ i = int(d.hashPrev[i&windowMask]) - d.hashOffset
354+ if i < minIndex || i < 0 {
355+ break
356+ }
357+ }
358+ return
359+}
360+
361+func (d *compressor) writeStoredBlock(buf []byte) error {
362+ if d.w.writeStoredHeader(len(buf), false); d.w.err != nil {
363+ return d.w.err
364+ }
365+ d.w.writeBytes(buf)
366+ return d.w.err
367+}
368+
369+const hashmul = 0x1e35a7bd
370+
371+// hash4 returns a hash representation of the first 4 bytes
372+// of the supplied slice.
373+// The caller must ensure that len(b) >= 4.
374+func hash4(b []byte) uint32 {
375+ return ((uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24) * hashmul) >> (32 - hashBits)
376+}
377+
378+// bulkHash4 will compute hashes using the same
379+// algorithm as hash4
380+func bulkHash4(b []byte, dst []uint32) {
381+ if len(b) < minMatchLength {
382+ return
383+ }
384+ hb := uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24
385+ dst[0] = (hb * hashmul) >> (32 - hashBits)
386+ end := len(b) - minMatchLength + 1
387+ for i := 1; i < end; i++ {
388+ hb = (hb << 8) | uint32(b[i+3])
389+ dst[i] = (hb * hashmul) >> (32 - hashBits)
390+ }
391+}
392+
393+// matchLen returns the number of matching bytes in a and b
394+// up to length 'max'. Both slices must be at least 'max'
395+// bytes in size.
396+func matchLen(a, b []byte, max int) int {
397+ a = a[:max]
398+ b = b[:len(a)]
399+ for i, av := range a {
400+ if b[i] != av {
401+ return i
402+ }
403+ }
404+ return max
405+}
406+
407+func (d *compressor) initDeflate() {
408+ d.window = make([]byte, 2*windowSize)
409+ d.hashOffset = 1
410+ d.length = minMatchLength - 1
411+ d.offset = 0
412+ d.byteAvailable = false
413+ d.index = 0
414+ d.hash = 0
415+ d.chainHead = -1
416+ d.bulkHasher = bulkHash4
417+ if useSSE42 {
418+ d.bulkHasher = crc32sseAll
419+ }
420+}
421+
422+// Assumes that d.fastSkipHashing != skipNever,
423+// otherwise use deflateLazy
424+func (d *compressor) deflate() {
425+
426+ // Sanity enables additional runtime tests.
427+ // It's intended to be used during development
428+ // to supplement the currently ad-hoc unit tests.
429+ const sanity = false
430+
431+ if d.windowEnd-d.index < minMatchLength+maxMatchLength && !d.sync {
432+ return
433+ }
434+
435+ d.maxInsertIndex = d.windowEnd - (minMatchLength - 1)
436+ if d.index < d.maxInsertIndex {
437+ d.hash = hash4(d.window[d.index : d.index+minMatchLength])
438+ }
439+
440+ for {
441+ if sanity && d.index > d.windowEnd {
442+ panic("index > windowEnd")
443+ }
444+ lookahead := d.windowEnd - d.index
445+ if lookahead < minMatchLength+maxMatchLength {
446+ if !d.sync {
447+ return
448+ }
449+ if sanity && d.index > d.windowEnd {
450+ panic("index > windowEnd")
451+ }
452+ if lookahead == 0 {
453+ if d.tokens.n > 0 {
454+ if d.err = d.writeBlockSkip(d.tokens, d.index, false); d.err != nil {
455+ return
456+ }
457+ d.tokens.n = 0
458+ }
459+ return
460+ }
461+ }
462+ if d.index < d.maxInsertIndex {
463+ // Update the hash
464+ d.hash = hash4(d.window[d.index : d.index+minMatchLength])
465+ ch := d.hashHead[d.hash&hashMask]
466+ d.chainHead = int(ch)
467+ d.hashPrev[d.index&windowMask] = ch
468+ d.hashHead[d.hash&hashMask] = uint32(d.index + d.hashOffset)
469+ }
470+ d.length = minMatchLength - 1
471+ d.offset = 0
472+ minIndex := d.index - windowSize
473+ if minIndex < 0 {
474+ minIndex = 0
475+ }
476+
477+ if d.chainHead-d.hashOffset >= minIndex && lookahead > minMatchLength-1 {
478+ if newLength, newOffset, ok := d.findMatch(d.index, d.chainHead-d.hashOffset, minMatchLength-1, lookahead); ok {
479+ d.length = newLength
480+ d.offset = newOffset
481+ }
482+ }
483+ if d.length >= minMatchLength {
484+ d.ii = 0
485+ // There was a match at the previous step, and the current match is
486+ // not better. Output the previous match.
487+ // "d.length-3" should NOT be "d.length-minMatchLength", since the format always assume 3
488+ d.tokens.tokens[d.tokens.n] = matchToken(uint32(d.length-3), uint32(d.offset-minOffsetSize))
489+ d.tokens.n++
490+ // Insert in the hash table all strings up to the end of the match.
491+ // index and index-1 are already inserted. If there is not enough
492+ // lookahead, the last two strings are not inserted into the hash
493+ // table.
494+ if d.length <= d.fastSkipHashing {
495+ var newIndex int
496+ newIndex = d.index + d.length
497+ // Calculate missing hashes
498+ end := newIndex
499+ if end > d.maxInsertIndex {
500+ end = d.maxInsertIndex
501+ }
502+ end += minMatchLength - 1
503+ startindex := d.index + 1
504+ if startindex > d.maxInsertIndex {
505+ startindex = d.maxInsertIndex
506+ }
507+ tocheck := d.window[startindex:end]
508+ dstSize := len(tocheck) - minMatchLength + 1
509+ if dstSize > 0 {
510+ dst := d.hashMatch[:dstSize]
511+ bulkHash4(tocheck, dst)
512+ var newH uint32
513+ for i, val := range dst {
514+ di := i + startindex
515+ newH = val & hashMask
516+ // Get previous value with the same hash.
517+ // Our chain should point to the previous value.
518+ d.hashPrev[di&windowMask] = d.hashHead[newH]
519+ // Set the head of the hash chain to us.
520+ d.hashHead[newH] = uint32(di + d.hashOffset)
521+ }
522+ d.hash = newH
523+ }
524+ d.index = newIndex
525+ } else {
526+ // For matches this long, we don't bother inserting each individual
527+ // item into the table.
528+ d.index += d.length
529+ if d.index < d.maxInsertIndex {
530+ d.hash = hash4(d.window[d.index : d.index+minMatchLength])
531+ }
532+ }
533+ if d.tokens.n == maxFlateBlockTokens {
534+ // The block includes the current character
535+ if d.err = d.writeBlockSkip(d.tokens, d.index, false); d.err != nil {
536+ return
537+ }
538+ d.tokens.n = 0
539+ }
540+ } else {
541+ d.ii++
542+ end := d.index + int(d.ii>>uint(d.fastSkipHashing)) + 1
543+ if end > d.windowEnd {
544+ end = d.windowEnd
545+ }
546+ for i := d.index; i < end; i++ {
547+ d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[i]))
548+ d.tokens.n++
549+ if d.tokens.n == maxFlateBlockTokens {
550+ if d.err = d.writeBlockSkip(d.tokens, i+1, false); d.err != nil {
551+ return
552+ }
553+ d.tokens.n = 0
554+ }
555+ }
556+ d.index = end
557+ }
558+ }
559+}
560+
561+// deflateLazy is the same as deflate, but with d.fastSkipHashing == skipNever,
562+// meaning it always has lazy matching on.
563+func (d *compressor) deflateLazy() {
564+ // Sanity enables additional runtime tests.
565+ // It's intended to be used during development
566+ // to supplement the currently ad-hoc unit tests.
567+ const sanity = false
568+
569+ if d.windowEnd-d.index < minMatchLength+maxMatchLength && !d.sync {
570+ return
571+ }
572+
573+ d.maxInsertIndex = d.windowEnd - (minMatchLength - 1)
574+ if d.index < d.maxInsertIndex {
575+ d.hash = hash4(d.window[d.index : d.index+minMatchLength])
576+ }
577+
578+ for {
579+ if sanity && d.index > d.windowEnd {
580+ panic("index > windowEnd")
581+ }
582+ lookahead := d.windowEnd - d.index
583+ if lookahead < minMatchLength+maxMatchLength {
584+ if !d.sync {
585+ return
586+ }
587+ if sanity && d.index > d.windowEnd {
588+ panic("index > windowEnd")
589+ }
590+ if lookahead == 0 {
591+ // Flush current output block if any.
592+ if d.byteAvailable {
593+ // There is still one pending token that needs to be flushed
594+ d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1]))
595+ d.tokens.n++
596+ d.byteAvailable = false
597+ }
598+ if d.tokens.n > 0 {
599+ if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil {
600+ return
601+ }
602+ d.tokens.n = 0
603+ }
604+ return
605+ }
606+ }
607+ if d.index < d.maxInsertIndex {
608+ // Update the hash
609+ d.hash = hash4(d.window[d.index : d.index+minMatchLength])
610+ ch := d.hashHead[d.hash&hashMask]
611+ d.chainHead = int(ch)
612+ d.hashPrev[d.index&windowMask] = ch
613+ d.hashHead[d.hash&hashMask] = uint32(d.index + d.hashOffset)
614+ }
615+ prevLength := d.length
616+ prevOffset := d.offset
617+ d.length = minMatchLength - 1
618+ d.offset = 0
619+ minIndex := d.index - windowSize
620+ if minIndex < 0 {
621+ minIndex = 0
622+ }
623+
624+ if d.chainHead-d.hashOffset >= minIndex && lookahead > prevLength && prevLength < d.lazy {
625+ if newLength, newOffset, ok := d.findMatch(d.index, d.chainHead-d.hashOffset, minMatchLength-1, lookahead); ok {
626+ d.length = newLength
627+ d.offset = newOffset
628+ }
629+ }
630+ if prevLength >= minMatchLength && d.length <= prevLength {
631+ // There was a match at the previous step, and the current match is
632+ // not better. Output the previous match.
633+ d.tokens.tokens[d.tokens.n] = matchToken(uint32(prevLength-3), uint32(prevOffset-minOffsetSize))
634+ d.tokens.n++
635+
636+ // Insert in the hash table all strings up to the end of the match.
637+ // index and index-1 are already inserted. If there is not enough
638+ // lookahead, the last two strings are not inserted into the hash
639+ // table.
640+ var newIndex int
641+ newIndex = d.index + prevLength - 1
642+ // Calculate missing hashes
643+ end := newIndex
644+ if end > d.maxInsertIndex {
645+ end = d.maxInsertIndex
646+ }
647+ end += minMatchLength - 1
648+ startindex := d.index + 1
649+ if startindex > d.maxInsertIndex {
650+ startindex = d.maxInsertIndex
651+ }
652+ tocheck := d.window[startindex:end]
653+ dstSize := len(tocheck) - minMatchLength + 1
654+ if dstSize > 0 {
655+ dst := d.hashMatch[:dstSize]
656+ bulkHash4(tocheck, dst)
657+ var newH uint32
658+ for i, val := range dst {
659+ di := i + startindex
660+ newH = val & hashMask
661+ // Get previous value with the same hash.
662+ // Our chain should point to the previous value.
663+ d.hashPrev[di&windowMask] = d.hashHead[newH]
664+ // Set the head of the hash chain to us.
665+ d.hashHead[newH] = uint32(di + d.hashOffset)
666+ }
667+ d.hash = newH
668+ }
669+
670+ d.index = newIndex
671+ d.byteAvailable = false
672+ d.length = minMatchLength - 1
673+ if d.tokens.n == maxFlateBlockTokens {
674+ // The block includes the current character
675+ if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil {
676+ return
677+ }
678+ d.tokens.n = 0
679+ }
680+ } else {
681+ // Reset, if we got a match this run.
682+ if d.length >= minMatchLength {
683+ d.ii = 0
684+ }
685+ // We have a byte waiting. Emit it.
686+ if d.byteAvailable {
687+ d.ii++
688+ d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1]))
689+ d.tokens.n++
690+ if d.tokens.n == maxFlateBlockTokens {
691+ if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil {
692+ return
693+ }
694+ d.tokens.n = 0
695+ }
696+ d.index++
697+
698+ // If we have a long run of no matches, skip additional bytes
699+ // Resets when d.ii overflows after 64KB.
700+ if d.ii > 31 {
701+ n := int(d.ii >> 5)
702+ for j := 0; j < n; j++ {
703+ if d.index >= d.windowEnd-1 {
704+ break
705+ }
706+
707+ d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1]))
708+ d.tokens.n++
709+ if d.tokens.n == maxFlateBlockTokens {
710+ if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil {
711+ return
712+ }
713+ d.tokens.n = 0
714+ }
715+ d.index++
716+ }
717+ // Flush last byte
718+ d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1]))
719+ d.tokens.n++
720+ d.byteAvailable = false
721+ // d.length = minMatchLength - 1 // not needed, since d.ii is reset above, so it should never be > minMatchLength
722+ if d.tokens.n == maxFlateBlockTokens {
723+ if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil {
724+ return
725+ }
726+ d.tokens.n = 0
727+ }
728+ }
729+ } else {
730+ d.index++
731+ d.byteAvailable = true
732+ }
733+ }
734+ }
735+}
736+
737+// Assumes that d.fastSkipHashing != skipNever,
738+// otherwise use deflateLazySSE
739+func (d *compressor) deflateSSE() {
740+
741+ // Sanity enables additional runtime tests.
742+ // It's intended to be used during development
743+ // to supplement the currently ad-hoc unit tests.
744+ const sanity = false
745+
746+ if d.windowEnd-d.index < minMatchLength+maxMatchLength && !d.sync {
747+ return
748+ }
749+
750+ d.maxInsertIndex = d.windowEnd - (minMatchLength - 1)
751+ if d.index < d.maxInsertIndex {
752+ d.hash = crc32sse(d.window[d.index:d.index+minMatchLength]) & hashMask
753+ }
754+
755+ for {
756+ if sanity && d.index > d.windowEnd {
757+ panic("index > windowEnd")
758+ }
759+ lookahead := d.windowEnd - d.index
760+ if lookahead < minMatchLength+maxMatchLength {
761+ if !d.sync {
762+ return
763+ }
764+ if sanity && d.index > d.windowEnd {
765+ panic("index > windowEnd")
766+ }
767+ if lookahead == 0 {
768+ if d.tokens.n > 0 {
769+ if d.err = d.writeBlockSkip(d.tokens, d.index, false); d.err != nil {
770+ return
771+ }
772+ d.tokens.n = 0
773+ }
774+ return
775+ }
776+ }
777+ if d.index < d.maxInsertIndex {
778+ // Update the hash
779+ d.hash = crc32sse(d.window[d.index:d.index+minMatchLength]) & hashMask
780+ ch := d.hashHead[d.hash]
781+ d.chainHead = int(ch)
782+ d.hashPrev[d.index&windowMask] = ch
783+ d.hashHead[d.hash] = uint32(d.index + d.hashOffset)
784+ }
785+ d.length = minMatchLength - 1
786+ d.offset = 0
787+ minIndex := d.index - windowSize
788+ if minIndex < 0 {
789+ minIndex = 0
790+ }
791+
792+ if d.chainHead-d.hashOffset >= minIndex && lookahead > minMatchLength-1 {
793+ if newLength, newOffset, ok := d.findMatchSSE(d.index, d.chainHead-d.hashOffset, minMatchLength-1, lookahead); ok {
794+ d.length = newLength
795+ d.offset = newOffset
796+ }
797+ }
798+ if d.length >= minMatchLength {
799+ d.ii = 0
800+ // There was a match at the previous step, and the current match is
801+ // not better. Output the previous match.
802+ // "d.length-3" should NOT be "d.length-minMatchLength", since the format always assume 3
803+ d.tokens.tokens[d.tokens.n] = matchToken(uint32(d.length-3), uint32(d.offset-minOffsetSize))
804+ d.tokens.n++
805+ // Insert in the hash table all strings up to the end of the match.
806+ // index and index-1 are already inserted. If there is not enough
807+ // lookahead, the last two strings are not inserted into the hash
808+ // table.
809+ if d.length <= d.fastSkipHashing {
810+ var newIndex int
811+ newIndex = d.index + d.length
812+ // Calculate missing hashes
813+ end := newIndex
814+ if end > d.maxInsertIndex {
815+ end = d.maxInsertIndex
816+ }
817+ end += minMatchLength - 1
818+ startindex := d.index + 1
819+ if startindex > d.maxInsertIndex {
820+ startindex = d.maxInsertIndex
821+ }
822+ tocheck := d.window[startindex:end]
823+ dstSize := len(tocheck) - minMatchLength + 1
824+ if dstSize > 0 {
825+ dst := d.hashMatch[:dstSize]
826+
827+ crc32sseAll(tocheck, dst)
828+ var newH uint32
829+ for i, val := range dst {
830+ di := i + startindex
831+ newH = val & hashMask
832+ // Get previous value with the same hash.
833+ // Our chain should point to the previous value.
834+ d.hashPrev[di&windowMask] = d.hashHead[newH]
835+ // Set the head of the hash chain to us.
836+ d.hashHead[newH] = uint32(di + d.hashOffset)
837+ }
838+ d.hash = newH
839+ }
840+ d.index = newIndex
841+ } else {
842+ // For matches this long, we don't bother inserting each individual
843+ // item into the table.
844+ d.index += d.length
845+ if d.index < d.maxInsertIndex {
846+ d.hash = crc32sse(d.window[d.index:d.index+minMatchLength]) & hashMask
847+ }
848+ }
849+ if d.tokens.n == maxFlateBlockTokens {
850+ // The block includes the current character
851+ if d.err = d.writeBlockSkip(d.tokens, d.index, false); d.err != nil {
852+ return
853+ }
854+ d.tokens.n = 0
855+ }
856+ } else {
857+ d.ii++
858+ end := d.index + int(d.ii>>5) + 1
859+ if end > d.windowEnd {
860+ end = d.windowEnd
861+ }
862+ for i := d.index; i < end; i++ {
863+ d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[i]))
864+ d.tokens.n++
865+ if d.tokens.n == maxFlateBlockTokens {
866+ if d.err = d.writeBlockSkip(d.tokens, i+1, false); d.err != nil {
867+ return
868+ }
869+ d.tokens.n = 0
870+ }
871+ }
872+ d.index = end
873+ }
874+ }
875+}
876+
877+// deflateLazy is the same as deflate, but with d.fastSkipHashing == skipNever,
878+// meaning it always has lazy matching on.
879+func (d *compressor) deflateLazySSE() {
880+ // Sanity enables additional runtime tests.
881+ // It's intended to be used during development
882+ // to supplement the currently ad-hoc unit tests.
883+ const sanity = false
884+
885+ if d.windowEnd-d.index < minMatchLength+maxMatchLength && !d.sync {
886+ return
887+ }
888+
889+ d.maxInsertIndex = d.windowEnd - (minMatchLength - 1)
890+ if d.index < d.maxInsertIndex {
891+ d.hash = crc32sse(d.window[d.index:d.index+minMatchLength]) & hashMask
892+ }
893+
894+ for {
895+ if sanity && d.index > d.windowEnd {
896+ panic("index > windowEnd")
897+ }
898+ lookahead := d.windowEnd - d.index
899+ if lookahead < minMatchLength+maxMatchLength {
900+ if !d.sync {
901+ return
902+ }
903+ if sanity && d.index > d.windowEnd {
904+ panic("index > windowEnd")
905+ }
906+ if lookahead == 0 {
907+ // Flush current output block if any.
908+ if d.byteAvailable {
909+ // There is still one pending token that needs to be flushed
910+ d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1]))
911+ d.tokens.n++
912+ d.byteAvailable = false
913+ }
914+ if d.tokens.n > 0 {
915+ if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil {
916+ return
917+ }
918+ d.tokens.n = 0
919+ }
920+ return
921+ }
922+ }
923+ if d.index < d.maxInsertIndex {
924+ // Update the hash
925+ d.hash = crc32sse(d.window[d.index:d.index+minMatchLength]) & hashMask
926+ ch := d.hashHead[d.hash]
927+ d.chainHead = int(ch)
928+ d.hashPrev[d.index&windowMask] = ch
929+ d.hashHead[d.hash] = uint32(d.index + d.hashOffset)
930+ }
931+ prevLength := d.length
932+ prevOffset := d.offset
933+ d.length = minMatchLength - 1
934+ d.offset = 0
935+ minIndex := d.index - windowSize
936+ if minIndex < 0 {
937+ minIndex = 0
938+ }
939+
940+ if d.chainHead-d.hashOffset >= minIndex && lookahead > prevLength && prevLength < d.lazy {
941+ if newLength, newOffset, ok := d.findMatchSSE(d.index, d.chainHead-d.hashOffset, minMatchLength-1, lookahead); ok {
942+ d.length = newLength
943+ d.offset = newOffset
944+ }
945+ }
946+ if prevLength >= minMatchLength && d.length <= prevLength {
947+ // There was a match at the previous step, and the current match is
948+ // not better. Output the previous match.
949+ d.tokens.tokens[d.tokens.n] = matchToken(uint32(prevLength-3), uint32(prevOffset-minOffsetSize))
950+ d.tokens.n++
951+
952+ // Insert in the hash table all strings up to the end of the match.
953+ // index and index-1 are already inserted. If there is not enough
954+ // lookahead, the last two strings are not inserted into the hash
955+ // table.
956+ var newIndex int
957+ newIndex = d.index + prevLength - 1
958+ // Calculate missing hashes
959+ end := newIndex
960+ if end > d.maxInsertIndex {
961+ end = d.maxInsertIndex
962+ }
963+ end += minMatchLength - 1
964+ startindex := d.index + 1
965+ if startindex > d.maxInsertIndex {
966+ startindex = d.maxInsertIndex
967+ }
968+ tocheck := d.window[startindex:end]
969+ dstSize := len(tocheck) - minMatchLength + 1
970+ if dstSize > 0 {
971+ dst := d.hashMatch[:dstSize]
972+ crc32sseAll(tocheck, dst)
973+ var newH uint32
974+ for i, val := range dst {
975+ di := i + startindex
976+ newH = val & hashMask
977+ // Get previous value with the same hash.
978+ // Our chain should point to the previous value.
979+ d.hashPrev[di&windowMask] = d.hashHead[newH]
980+ // Set the head of the hash chain to us.
981+ d.hashHead[newH] = uint32(di + d.hashOffset)
982+ }
983+ d.hash = newH
984+ }
985+
986+ d.index = newIndex
987+ d.byteAvailable = false
988+ d.length = minMatchLength - 1
989+ if d.tokens.n == maxFlateBlockTokens {
990+ // The block includes the current character
991+ if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil {
992+ return
993+ }
994+ d.tokens.n = 0
995+ }
996+ } else {
997+ // Reset, if we got a match this run.
998+ if d.length >= minMatchLength {
999+ d.ii = 0
1000+ }
1001+ // We have a byte waiting. Emit it.
1002+ if d.byteAvailable {
1003+ d.ii++
1004+ d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1]))
1005+ d.tokens.n++
1006+ if d.tokens.n == maxFlateBlockTokens {
1007+ if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil {
1008+ return
1009+ }
1010+ d.tokens.n = 0
1011+ }
1012+ d.index++
1013+
1014+ // If we have a long run of no matches, skip additional bytes
1015+ // Resets when d.ii overflows after 64KB.
1016+ if d.ii > 31 {
1017+ n := int(d.ii >> 6)
1018+ for j := 0; j < n; j++ {
1019+ if d.index >= d.windowEnd-1 {
1020+ break
1021+ }
1022+
1023+ d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1]))
1024+ d.tokens.n++
1025+ if d.tokens.n == maxFlateBlockTokens {
1026+ if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil {
1027+ return
1028+ }
1029+ d.tokens.n = 0
1030+ }
1031+ d.index++
1032+ }
1033+ // Flush last byte
1034+ d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1]))
1035+ d.tokens.n++
1036+ d.byteAvailable = false
1037+ // d.length = minMatchLength - 1 // not needed, since d.ii is reset above, so it should never be > minMatchLength
1038+ if d.tokens.n == maxFlateBlockTokens {
1039+ if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil {
1040+ return
1041+ }
1042+ d.tokens.n = 0
1043+ }
1044+ }
1045+ } else {
1046+ d.index++
1047+ d.byteAvailable = true
1048+ }
1049+ }
1050+ }
1051+}
1052+
1053+func (d *compressor) store() {
1054+ if d.windowEnd > 0 && (d.windowEnd == maxStoreBlockSize || d.sync) {
1055+ d.err = d.writeStoredBlock(d.window[:d.windowEnd])
1056+ d.windowEnd = 0
1057+ }
1058+}
1059+
1060+// fillWindow will fill the buffer with data for huffman-only compression.
1061+// The number of bytes copied is returned.
1062+func (d *compressor) fillBlock(b []byte) int {
1063+ n := copy(d.window[d.windowEnd:], b)
1064+ d.windowEnd += n
1065+ return n
1066+}
1067+
1068+// storeHuff will compress and store the currently added data,
1069+// if enough has been accumulated or we at the end of the stream.
1070+// Any error that occurred will be in d.err
1071+func (d *compressor) storeHuff() {
1072+ if d.windowEnd < len(d.window) && !d.sync || d.windowEnd == 0 {
1073+ return
1074+ }
1075+ d.w.writeBlockHuff(false, d.window[:d.windowEnd])
1076+ d.err = d.w.err
1077+ d.windowEnd = 0
1078+}
1079+
1080+// storeHuff will compress and store the currently added data,
1081+// if enough has been accumulated or we at the end of the stream.
1082+// Any error that occurred will be in d.err
1083+func (d *compressor) storeSnappy() {
1084+ // We only compress if we have maxStoreBlockSize.
1085+ if d.windowEnd < maxStoreBlockSize {
1086+ if !d.sync {
1087+ return
1088+ }
1089+ // Handle extremely small sizes.
1090+ if d.windowEnd < 128 {
1091+ if d.windowEnd == 0 {
1092+ return
1093+ }
1094+ if d.windowEnd <= 32 {
1095+ d.err = d.writeStoredBlock(d.window[:d.windowEnd])
1096+ d.tokens.n = 0
1097+ d.windowEnd = 0
1098+ } else {
1099+ d.w.writeBlockHuff(false, d.window[:d.windowEnd])
1100+ d.err = d.w.err
1101+ }
1102+ d.tokens.n = 0
1103+ d.windowEnd = 0
1104+ d.snap.Reset()
1105+ return
1106+ }
1107+ }
1108+
1109+ d.snap.Encode(&d.tokens, d.window[:d.windowEnd])
1110+ // If we made zero matches, store the block as is.
1111+ if int(d.tokens.n) == d.windowEnd {
1112+ d.err = d.writeStoredBlock(d.window[:d.windowEnd])
1113+ // If we removed less than 1/16th, huffman compress the block.
1114+ } else if int(d.tokens.n) > d.windowEnd-(d.windowEnd>>4) {
1115+ d.w.writeBlockHuff(false, d.window[:d.windowEnd])
1116+ d.err = d.w.err
1117+ } else {
1118+ d.w.writeBlockDynamic(d.tokens.tokens[:d.tokens.n], false, d.window[:d.windowEnd])
1119+ d.err = d.w.err
1120+ }
1121+ d.tokens.n = 0
1122+ d.windowEnd = 0
1123+}
1124+
1125+// write will add input byte to the stream.
1126+// Unless an error occurs all bytes will be consumed.
1127+func (d *compressor) write(b []byte) (n int, err error) {
1128+ if d.err != nil {
1129+ return 0, d.err
1130+ }
1131+ n = len(b)
1132+ for len(b) > 0 {
1133+ d.step(d)
1134+ b = b[d.fill(d, b):]
1135+ if d.err != nil {
1136+ return 0, d.err
1137+ }
1138+ }
1139+ return n, d.err
1140+}
1141+
1142+func (d *compressor) syncFlush() error {
1143+ d.sync = true
1144+ if d.err != nil {
1145+ return d.err
1146+ }
1147+ d.step(d)
1148+ if d.err == nil {
1149+ d.w.writeStoredHeader(0, false)
1150+ d.w.flush()
1151+ d.err = d.w.err
1152+ }
1153+ d.sync = false
1154+ return d.err
1155+}
1156+
1157+func (d *compressor) init(w io.Writer, level int) (err error) {
1158+ d.w = newHuffmanBitWriter(w)
1159+
1160+ switch {
1161+ case level == NoCompression:
1162+ d.window = make([]byte, maxStoreBlockSize)
1163+ d.fill = (*compressor).fillBlock
1164+ d.step = (*compressor).store
1165+ case level == ConstantCompression:
1166+ d.window = make([]byte, maxStoreBlockSize)
1167+ d.fill = (*compressor).fillBlock
1168+ d.step = (*compressor).storeHuff
1169+ case level >= 1 && level <= 4:
1170+ d.snap = newSnappy(level)
1171+ d.window = make([]byte, maxStoreBlockSize)
1172+ d.fill = (*compressor).fillBlock
1173+ d.step = (*compressor).storeSnappy
1174+ case level == DefaultCompression:
1175+ level = 5
1176+ fallthrough
1177+ case 5 <= level && level <= 9:
1178+ d.compressionLevel = levels[level]
1179+ d.initDeflate()
1180+ d.fill = (*compressor).fillDeflate
1181+ if d.fastSkipHashing == skipNever {
1182+ if useSSE42 {
1183+ d.step = (*compressor).deflateLazySSE
1184+ } else {
1185+ d.step = (*compressor).deflateLazy
1186+ }
1187+ } else {
1188+ if useSSE42 {
1189+ d.step = (*compressor).deflateSSE
1190+ } else {
1191+ d.step = (*compressor).deflate
1192+
1193+ }
1194+ }
1195+ default:
1196+ return fmt.Errorf("flate: invalid compression level %d: want value in range [-2, 9]", level)
1197+ }
1198+ return nil
1199+}
1200+
1201+// reset the state of the compressor.
1202+func (d *compressor) reset(w io.Writer) {
1203+ d.w.reset(w)
1204+ d.sync = false
1205+ d.err = nil
1206+ // We only need to reset a few things for Snappy.
1207+ if d.snap != nil {
1208+ d.snap.Reset()
1209+ d.windowEnd = 0
1210+ d.tokens.n = 0
1211+ return
1212+ }
1213+ switch d.compressionLevel.chain {
1214+ case 0:
1215+ // level was NoCompression or ConstantCompresssion.
1216+ d.windowEnd = 0
1217+ default:
1218+ d.chainHead = -1
1219+ for i := range d.hashHead {
1220+ d.hashHead[i] = 0
1221+ }
1222+ for i := range d.hashPrev {
1223+ d.hashPrev[i] = 0
1224+ }
1225+ d.hashOffset = 1
1226+ d.index, d.windowEnd = 0, 0
1227+ d.blockStart, d.byteAvailable = 0, false
1228+ d.tokens.n = 0
1229+ d.length = minMatchLength - 1
1230+ d.offset = 0
1231+ d.hash = 0
1232+ d.ii = 0
1233+ d.maxInsertIndex = 0
1234+ }
1235+}
1236+
1237+func (d *compressor) close() error {
1238+ if d.err != nil {
1239+ return d.err
1240+ }
1241+ d.sync = true
1242+ d.step(d)
1243+ if d.err != nil {
1244+ return d.err
1245+ }
1246+ if d.w.writeStoredHeader(0, true); d.w.err != nil {
1247+ return d.w.err
1248+ }
1249+ d.w.flush()
1250+ return d.w.err
1251+}
1252+
1253+// NewWriter returns a new Writer compressing data at the given level.
1254+// Following zlib, levels range from 1 (BestSpeed) to 9 (BestCompression);
1255+// higher levels typically run slower but compress more.
1256+// Level 0 (NoCompression) does not attempt any compression; it only adds the
1257+// necessary DEFLATE framing.
1258+// Level -1 (DefaultCompression) uses the default compression level.
1259+// Level -2 (ConstantCompression) will use Huffman compression only, giving
1260+// a very fast compression for all types of input, but sacrificing considerable
1261+// compression efficiency.
1262+//
1263+// If level is in the range [-2, 9] then the error returned will be nil.
1264+// Otherwise the error returned will be non-nil.
1265+func NewWriter(w io.Writer, level int) (*Writer, error) {
1266+ var dw Writer
1267+ if err := dw.d.init(w, level); err != nil {
1268+ return nil, err
1269+ }
1270+ return &dw, nil
1271+}
1272+
1273+// NewWriterDict is like NewWriter but initializes the new
1274+// Writer with a preset dictionary. The returned Writer behaves
1275+// as if the dictionary had been written to it without producing
1276+// any compressed output. The compressed data written to w
1277+// can only be decompressed by a Reader initialized with the
1278+// same dictionary.
1279+func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, error) {
1280+ dw := &dictWriter{w}
1281+ zw, err := NewWriter(dw, level)
1282+ if err != nil {
1283+ return nil, err
1284+ }
1285+ zw.d.fillWindow(dict)
1286+ zw.dict = append(zw.dict, dict...) // duplicate dictionary for Reset method.
1287+ return zw, err
1288+}
1289+
1290+type dictWriter struct {
1291+ w io.Writer
1292+}
1293+
1294+func (w *dictWriter) Write(b []byte) (n int, err error) {
1295+ return w.w.Write(b)
1296+}
1297+
1298+// A Writer takes data written to it and writes the compressed
1299+// form of that data to an underlying writer (see NewWriter).
1300+type Writer struct {
1301+ d compressor
1302+ dict []byte
1303+}
1304+
1305+// Write writes data to w, which will eventually write the
1306+// compressed form of data to its underlying writer.
1307+func (w *Writer) Write(data []byte) (n int, err error) {
1308+ return w.d.write(data)
1309+}
1310+
1311+// Flush flushes any pending data to the underlying writer.
1312+// It is useful mainly in compressed network protocols, to ensure that
1313+// a remote reader has enough data to reconstruct a packet.
1314+// Flush does not return until the data has been written.
1315+// Calling Flush when there is no pending data still causes the Writer
1316+// to emit a sync marker of at least 4 bytes.
1317+// If the underlying writer returns an error, Flush returns that error.
1318+//
1319+// In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH.
1320+func (w *Writer) Flush() error {
1321+ // For more about flushing:
1322+ // http://www.bolet.org/~pornin/deflate-flush.html
1323+ return w.d.syncFlush()
1324+}
1325+
1326+// Close flushes and closes the writer.
1327+func (w *Writer) Close() error {
1328+ return w.d.close()
1329+}
1330+
1331+// Reset discards the writer's state and makes it equivalent to
1332+// the result of NewWriter or NewWriterDict called with dst
1333+// and w's level and dictionary.
1334+func (w *Writer) Reset(dst io.Writer) {
1335+ if dw, ok := w.d.w.writer.(*dictWriter); ok {
1336+ // w was created with NewWriterDict
1337+ dw.w = dst
1338+ w.d.reset(dw)
1339+ w.d.fillWindow(w.dict)
1340+ } else {
1341+ // w was created with NewWriter
1342+ w.d.reset(dst)
1343+ }
1344+}
1345+
1346+// ResetDict discards the writer's state and makes it equivalent to
1347+// the result of NewWriter or NewWriterDict called with dst
1348+// and w's level, but sets a specific dictionary.
1349+func (w *Writer) ResetDict(dst io.Writer, dict []byte) {
1350+ w.dict = dict
1351+ w.d.reset(dst)
1352+ w.d.fillWindow(w.dict)
1353+}
vendor/github.com/klauspost/compress/flate/dict_decoder.go+184−0
@@ -0,0 +1,184 @@
1+// Copyright 2016 The Go Authors. All rights reserved.
2+// Use of this source code is governed by a BSD-style
3+// license that can be found in the LICENSE file.
4+
5+package flate
6+
7+// dictDecoder implements the LZ77 sliding dictionary as used in decompression.
8+// LZ77 decompresses data through sequences of two forms of commands:
9+//
10+// * Literal insertions: Runs of one or more symbols are inserted into the data
11+// stream as is. This is accomplished through the writeByte method for a
12+// single symbol, or combinations of writeSlice/writeMark for multiple symbols.
13+// Any valid stream must start with a literal insertion if no preset dictionary
14+// is used.
15+//
16+// * Backward copies: Runs of one or more symbols are copied from previously
17+// emitted data. Backward copies come as the tuple (dist, length) where dist
18+// determines how far back in the stream to copy from and length determines how
19+// many bytes to copy. Note that it is valid for the length to be greater than
20+// the distance. Since LZ77 uses forward copies, that situation is used to
21+// perform a form of run-length encoding on repeated runs of symbols.
22+// The writeCopy and tryWriteCopy are used to implement this command.
23+//
24+// For performance reasons, this implementation performs little to no sanity
25+// checks about the arguments. As such, the invariants documented for each
26+// method call must be respected.
27+type dictDecoder struct {
28+ hist []byte // Sliding window history
29+
30+ // Invariant: 0 <= rdPos <= wrPos <= len(hist)
31+ wrPos int // Current output position in buffer
32+ rdPos int // Have emitted hist[:rdPos] already
33+ full bool // Has a full window length been written yet?
34+}
35+
36+// init initializes dictDecoder to have a sliding window dictionary of the given
37+// size. If a preset dict is provided, it will initialize the dictionary with
38+// the contents of dict.
39+func (dd *dictDecoder) init(size int, dict []byte) {
40+ *dd = dictDecoder{hist: dd.hist}
41+
42+ if cap(dd.hist) < size {
43+ dd.hist = make([]byte, size)
44+ }
45+ dd.hist = dd.hist[:size]
46+
47+ if len(dict) > len(dd.hist) {
48+ dict = dict[len(dict)-len(dd.hist):]
49+ }
50+ dd.wrPos = copy(dd.hist, dict)
51+ if dd.wrPos == len(dd.hist) {
52+ dd.wrPos = 0
53+ dd.full = true
54+ }
55+ dd.rdPos = dd.wrPos
56+}
57+
58+// histSize reports the total amount of historical data in the dictionary.
59+func (dd *dictDecoder) histSize() int {
60+ if dd.full {
61+ return len(dd.hist)
62+ }
63+ return dd.wrPos
64+}
65+
66+// availRead reports the number of bytes that can be flushed by readFlush.
67+func (dd *dictDecoder) availRead() int {
68+ return dd.wrPos - dd.rdPos
69+}
70+
71+// availWrite reports the available amount of output buffer space.
72+func (dd *dictDecoder) availWrite() int {
73+ return len(dd.hist) - dd.wrPos
74+}
75+
76+// writeSlice returns a slice of the available buffer to write data to.
77+//
78+// This invariant will be kept: len(s) <= availWrite()
79+func (dd *dictDecoder) writeSlice() []byte {
80+ return dd.hist[dd.wrPos:]
81+}
82+
83+// writeMark advances the writer pointer by cnt.
84+//
85+// This invariant must be kept: 0 <= cnt <= availWrite()
86+func (dd *dictDecoder) writeMark(cnt int) {
87+ dd.wrPos += cnt
88+}
89+
90+// writeByte writes a single byte to the dictionary.
91+//
92+// This invariant must be kept: 0 < availWrite()
93+func (dd *dictDecoder) writeByte(c byte) {
94+ dd.hist[dd.wrPos] = c
95+ dd.wrPos++
96+}
97+
98+// writeCopy copies a string at a given (dist, length) to the output.
99+// This returns the number of bytes copied and may be less than the requested
100+// length if the available space in the output buffer is too small.
101+//
102+// This invariant must be kept: 0 < dist <= histSize()
103+func (dd *dictDecoder) writeCopy(dist, length int) int {
104+ dstBase := dd.wrPos
105+ dstPos := dstBase
106+ srcPos := dstPos - dist
107+ endPos := dstPos + length
108+ if endPos > len(dd.hist) {
109+ endPos = len(dd.hist)
110+ }
111+
112+ // Copy non-overlapping section after destination position.
113+ //
114+ // This section is non-overlapping in that the copy length for this section
115+ // is always less than or equal to the backwards distance. This can occur
116+ // if a distance refers to data that wraps-around in the buffer.
117+ // Thus, a backwards copy is performed here; that is, the exact bytes in
118+ // the source prior to the copy is placed in the destination.
119+ if srcPos < 0 {
120+ srcPos += len(dd.hist)
121+ dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:])
122+ srcPos = 0
123+ }
124+
125+ // Copy possibly overlapping section before destination position.
126+ //
127+ // This section can overlap if the copy length for this section is larger
128+ // than the backwards distance. This is allowed by LZ77 so that repeated
129+ // strings can be succinctly represented using (dist, length) pairs.
130+ // Thus, a forwards copy is performed here; that is, the bytes copied is
131+ // possibly dependent on the resulting bytes in the destination as the copy
132+ // progresses along. This is functionally equivalent to the following:
133+ //
134+ // for i := 0; i < endPos-dstPos; i++ {
135+ // dd.hist[dstPos+i] = dd.hist[srcPos+i]
136+ // }
137+ // dstPos = endPos
138+ //
139+ for dstPos < endPos {
140+ dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:dstPos])
141+ }
142+
143+ dd.wrPos = dstPos
144+ return dstPos - dstBase
145+}
146+
147+// tryWriteCopy tries to copy a string at a given (distance, length) to the
148+// output. This specialized version is optimized for short distances.
149+//
150+// This method is designed to be inlined for performance reasons.
151+//
152+// This invariant must be kept: 0 < dist <= histSize()
153+func (dd *dictDecoder) tryWriteCopy(dist, length int) int {
154+ dstPos := dd.wrPos
155+ endPos := dstPos + length
156+ if dstPos < dist || endPos > len(dd.hist) {
157+ return 0
158+ }
159+ dstBase := dstPos
160+ srcPos := dstPos - dist
161+
162+ // Copy possibly overlapping section before destination position.
163+loop:
164+ dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:dstPos])
165+ if dstPos < endPos {
166+ goto loop // Avoid for-loop so that this function can be inlined
167+ }
168+
169+ dd.wrPos = dstPos
170+ return dstPos - dstBase
171+}
172+
173+// readFlush returns a slice of the historical buffer that is ready to be
174+// emitted to the user. The data returned by readFlush must be fully consumed
175+// before calling any other dictDecoder methods.
176+func (dd *dictDecoder) readFlush() []byte {
177+ toRead := dd.hist[dd.rdPos:dd.wrPos]
178+ dd.rdPos = dd.wrPos
179+ if dd.wrPos == len(dd.hist) {
180+ dd.wrPos, dd.rdPos = 0, 0
181+ dd.full = true
182+ }
183+ return toRead
184+}
vendor/github.com/klauspost/compress/flate/gen.go+265−0
@@ -0,0 +1,265 @@
1+// Copyright 2012 The Go Authors. All rights reserved.
2+// Use of this source code is governed by a BSD-style
3+// license that can be found in the LICENSE file.
4+
5+// +build ignore
6+
7+// This program generates fixedhuff.go
8+// Invoke as
9+//
10+// go run gen.go -output fixedhuff.go
11+
12+package main
13+
14+import (
15+ "bytes"
16+ "flag"
17+ "fmt"
18+ "go/format"
19+ "io/ioutil"
20+ "log"
21+)
22+
23+var filename = flag.String("output", "fixedhuff.go", "output file name")
24+
25+const maxCodeLen = 16
26+
27+// Note: the definition of the huffmanDecoder struct is copied from
28+// inflate.go, as it is private to the implementation.
29+
30+// chunk & 15 is number of bits
31+// chunk >> 4 is value, including table link
32+
33+const (
34+ huffmanChunkBits = 9
35+ huffmanNumChunks = 1 << huffmanChunkBits
36+ huffmanCountMask = 15
37+ huffmanValueShift = 4
38+)
39+
40+type huffmanDecoder struct {
41+ min int // the minimum code length
42+ chunks [huffmanNumChunks]uint32 // chunks as described above
43+ links [][]uint32 // overflow links
44+ linkMask uint32 // mask the width of the link table
45+}
46+
47+// Initialize Huffman decoding tables from array of code lengths.
48+// Following this function, h is guaranteed to be initialized into a complete
49+// tree (i.e., neither over-subscribed nor under-subscribed). The exception is a
50+// degenerate case where the tree has only a single symbol with length 1. Empty
51+// trees are permitted.
52+func (h *huffmanDecoder) init(bits []int) bool {
53+ // Sanity enables additional runtime tests during Huffman
54+ // table construction. It's intended to be used during
55+ // development to supplement the currently ad-hoc unit tests.
56+ const sanity = false
57+
58+ if h.min != 0 {
59+ *h = huffmanDecoder{}
60+ }
61+
62+ // Count number of codes of each length,
63+ // compute min and max length.
64+ var count [maxCodeLen]int
65+ var min, max int
66+ for _, n := range bits {
67+ if n == 0 {
68+ continue
69+ }
70+ if min == 0 || n < min {
71+ min = n
72+ }
73+ if n > max {
74+ max = n
75+ }
76+ count[n]++
77+ }
78+
79+ // Empty tree. The decompressor.huffSym function will fail later if the tree
80+ // is used. Technically, an empty tree is only valid for the HDIST tree and
81+ // not the HCLEN and HLIT tree. However, a stream with an empty HCLEN tree
82+ // is guaranteed to fail since it will attempt to use the tree to decode the
83+ // codes for the HLIT and HDIST trees. Similarly, an empty HLIT tree is
84+ // guaranteed to fail later since the compressed data section must be
85+ // composed of at least one symbol (the end-of-block marker).
86+ if max == 0 {
87+ return true
88+ }
89+
90+ code := 0
91+ var nextcode [maxCodeLen]int
92+ for i := min; i <= max; i++ {
93+ code <<= 1
94+ nextcode[i] = code
95+ code += count[i]
96+ }
97+
98+ // Check that the coding is complete (i.e., that we've
99+ // assigned all 2-to-the-max possible bit sequences).
100+ // Exception: To be compatible with zlib, we also need to
101+ // accept degenerate single-code codings. See also
102+ // TestDegenerateHuffmanCoding.
103+ if code != 1<<uint(max) && !(code == 1 && max == 1) {
104+ return false
105+ }
106+
107+ h.min = min
108+ if max > huffmanChunkBits {
109+ numLinks := 1 << (uint(max) - huffmanChunkBits)
110+ h.linkMask = uint32(numLinks - 1)
111+
112+ // create link tables
113+ link := nextcode[huffmanChunkBits+1] >> 1
114+ h.links = make([][]uint32, huffmanNumChunks-link)
115+ for j := uint(link); j < huffmanNumChunks; j++ {
116+ reverse := int(reverseByte[j>>8]) | int(reverseByte[j&0xff])<<8
117+ reverse >>= uint(16 - huffmanChunkBits)
118+ off := j - uint(link)
119+ if sanity && h.chunks[reverse] != 0 {
120+ panic("impossible: overwriting existing chunk")
121+ }
122+ h.chunks[reverse] = uint32(off<<huffmanValueShift | (huffmanChunkBits + 1))
123+ h.links[off] = make([]uint32, numLinks)
124+ }
125+ }
126+
127+ for i, n := range bits {
128+ if n == 0 {
129+ continue
130+ }
131+ code := nextcode[n]
132+ nextcode[n]++
133+ chunk := uint32(i<<huffmanValueShift | n)
134+ reverse := int(reverseByte[code>>8]) | int(reverseByte[code&0xff])<<8
135+ reverse >>= uint(16 - n)
136+ if n <= huffmanChunkBits {
137+ for off := reverse; off < len(h.chunks); off += 1 << uint(n) {
138+ // We should never need to overwrite
139+ // an existing chunk. Also, 0 is
140+ // never a valid chunk, because the
141+ // lower 4 "count" bits should be
142+ // between 1 and 15.
143+ if sanity && h.chunks[off] != 0 {
144+ panic("impossible: overwriting existing chunk")
145+ }
146+ h.chunks[off] = chunk
147+ }
148+ } else {
149+ j := reverse & (huffmanNumChunks - 1)
150+ if sanity && h.chunks[j]&huffmanCountMask != huffmanChunkBits+1 {
151+ // Longer codes should have been
152+ // associated with a link table above.
153+ panic("impossible: not an indirect chunk")
154+ }
155+ value := h.chunks[j] >> huffmanValueShift
156+ linktab := h.links[value]
157+ reverse >>= huffmanChunkBits
158+ for off := reverse; off < len(linktab); off += 1 << uint(n-huffmanChunkBits) {
159+ if sanity && linktab[off] != 0 {
160+ panic("impossible: overwriting existing chunk")
161+ }
162+ linktab[off] = chunk
163+ }
164+ }
165+ }
166+
167+ if sanity {
168+ // Above we've sanity checked that we never overwrote
169+ // an existing entry. Here we additionally check that
170+ // we filled the tables completely.
171+ for i, chunk := range h.chunks {
172+ if chunk == 0 {
173+ // As an exception, in the degenerate
174+ // single-code case, we allow odd
175+ // chunks to be missing.
176+ if code == 1 && i%2 == 1 {
177+ continue
178+ }
179+ panic("impossible: missing chunk")
180+ }
181+ }
182+ for _, linktab := range h.links {
183+ for _, chunk := range linktab {
184+ if chunk == 0 {
185+ panic("impossible: missing chunk")
186+ }
187+ }
188+ }
189+ }
190+
191+ return true
192+}
193+
194+func main() {
195+ flag.Parse()
196+
197+ var h huffmanDecoder
198+ var bits [288]int
199+ initReverseByte()
200+ for i := 0; i < 144; i++ {
201+ bits[i] = 8
202+ }
203+ for i := 144; i < 256; i++ {
204+ bits[i] = 9
205+ }
206+ for i := 256; i < 280; i++ {
207+ bits[i] = 7
208+ }
209+ for i := 280; i < 288; i++ {
210+ bits[i] = 8
211+ }
212+ h.init(bits[:])
213+ if h.links != nil {
214+ log.Fatal("Unexpected links table in fixed Huffman decoder")
215+ }
216+
217+ var buf bytes.Buffer
218+
219+ fmt.Fprintf(&buf, `// Copyright 2013 The Go Authors. All rights reserved.
220+// Use of this source code is governed by a BSD-style
221+// license that can be found in the LICENSE file.`+"\n\n")
222+
223+ fmt.Fprintln(&buf, "package flate")
224+ fmt.Fprintln(&buf)
225+ fmt.Fprintln(&buf, "// autogenerated by go run gen.go -output fixedhuff.go, DO NOT EDIT")
226+ fmt.Fprintln(&buf)
227+ fmt.Fprintln(&buf, "var fixedHuffmanDecoder = huffmanDecoder{")
228+ fmt.Fprintf(&buf, "\t%d,\n", h.min)
229+ fmt.Fprintln(&buf, "\t[huffmanNumChunks]uint32{")
230+ for i := 0; i < huffmanNumChunks; i++ {
231+ if i&7 == 0 {
232+ fmt.Fprintf(&buf, "\t\t")
233+ } else {
234+ fmt.Fprintf(&buf, " ")
235+ }
236+ fmt.Fprintf(&buf, "0x%04x,", h.chunks[i])
237+ if i&7 == 7 {
238+ fmt.Fprintln(&buf)
239+ }
240+ }
241+ fmt.Fprintln(&buf, "\t},")
242+ fmt.Fprintln(&buf, "\tnil, 0,")
243+ fmt.Fprintln(&buf, "}")
244+
245+ data, err := format.Source(buf.Bytes())
246+ if err != nil {
247+ log.Fatal(err)
248+ }
249+ err = ioutil.WriteFile(*filename, data, 0644)
250+ if err != nil {
251+ log.Fatal(err)
252+ }
253+}
254+
255+var reverseByte [256]byte
256+
257+func initReverseByte() {
258+ for x := 0; x < 256; x++ {
259+ var result byte
260+ for i := uint(0); i < 8; i++ {
261+ result |= byte(((x >> i) & 1) << (7 - i))
262+ }
263+ reverseByte[x] = result
264+ }
265+}
vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go+701−0
@@ -0,0 +1,701 @@
1+// Copyright 2009 The Go Authors. All rights reserved.
2+// Use of this source code is governed by a BSD-style
3+// license that can be found in the LICENSE file.
4+
5+package flate
6+
7+import (
8+ "io"
9+)
10+
11+const (
12+ // The largest offset code.
13+ offsetCodeCount = 30
14+
15+ // The special code used to mark the end of a block.
16+ endBlockMarker = 256
17+
18+ // The first length code.
19+ lengthCodesStart = 257
20+
21+ // The number of codegen codes.
22+ codegenCodeCount = 19
23+ badCode = 255
24+
25+ // bufferFlushSize indicates the buffer size
26+ // after which bytes are flushed to the writer.
27+ // Should preferably be a multiple of 6, since
28+ // we accumulate 6 bytes between writes to the buffer.
29+ bufferFlushSize = 240
30+
31+ // bufferSize is the actual output byte buffer size.
32+ // It must have additional headroom for a flush
33+ // which can contain up to 8 bytes.
34+ bufferSize = bufferFlushSize + 8
35+)
36+
37+// The number of extra bits needed by length code X - LENGTH_CODES_START.
38+var lengthExtraBits = []int8{
39+ /* 257 */ 0, 0, 0,
40+ /* 260 */ 0, 0, 0, 0, 0, 1, 1, 1, 1, 2,
41+ /* 270 */ 2, 2, 2, 3, 3, 3, 3, 4, 4, 4,
42+ /* 280 */ 4, 5, 5, 5, 5, 0,
43+}
44+
45+// The length indicated by length code X - LENGTH_CODES_START.
46+var lengthBase = []uint32{
47+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 10,
48+ 12, 14, 16, 20, 24, 28, 32, 40, 48, 56,
49+ 64, 80, 96, 112, 128, 160, 192, 224, 255,
50+}
51+
52+// offset code word extra bits.
53+var offsetExtraBits = []int8{
54+ 0, 0, 0, 0, 1, 1, 2, 2, 3, 3,
55+ 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
56+ 9, 9, 10, 10, 11, 11, 12, 12, 13, 13,
57+ /* extended window */
58+ 14, 14, 15, 15, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20,
59+}
60+
61+var offsetBase = []uint32{
62+ /* normal deflate */
63+ 0x000000, 0x000001, 0x000002, 0x000003, 0x000004,
64+ 0x000006, 0x000008, 0x00000c, 0x000010, 0x000018,
65+ 0x000020, 0x000030, 0x000040, 0x000060, 0x000080,
66+ 0x0000c0, 0x000100, 0x000180, 0x000200, 0x000300,
67+ 0x000400, 0x000600, 0x000800, 0x000c00, 0x001000,
68+ 0x001800, 0x002000, 0x003000, 0x004000, 0x006000,
69+
70+ /* extended window */
71+ 0x008000, 0x00c000, 0x010000, 0x018000, 0x020000,
72+ 0x030000, 0x040000, 0x060000, 0x080000, 0x0c0000,
73+ 0x100000, 0x180000, 0x200000, 0x300000,
74+}
75+
76+// The odd order in which the codegen code sizes are written.
77+var codegenOrder = []uint32{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}
78+
79+type huffmanBitWriter struct {
80+ // writer is the underlying writer.
81+ // Do not use it directly; use the write method, which ensures
82+ // that Write errors are sticky.
83+ writer io.Writer
84+
85+ // Data waiting to be written is bytes[0:nbytes]
86+ // and then the low nbits of bits.
87+ bits uint64
88+ nbits uint
89+ bytes [bufferSize]byte
90+ codegenFreq [codegenCodeCount]int32
91+ nbytes int
92+ literalFreq []int32
93+ offsetFreq []int32
94+ codegen []uint8
95+ literalEncoding *huffmanEncoder
96+ offsetEncoding *huffmanEncoder
97+ codegenEncoding *huffmanEncoder
98+ err error
99+}
100+
101+func newHuffmanBitWriter(w io.Writer) *huffmanBitWriter {
102+ return &huffmanBitWriter{
103+ writer: w,
104+ literalFreq: make([]int32, maxNumLit),
105+ offsetFreq: make([]int32, offsetCodeCount),
106+ codegen: make([]uint8, maxNumLit+offsetCodeCount+1),
107+ literalEncoding: newHuffmanEncoder(maxNumLit),
108+ codegenEncoding: newHuffmanEncoder(codegenCodeCount),
109+ offsetEncoding: newHuffmanEncoder(offsetCodeCount),
110+ }
111+}
112+
113+func (w *huffmanBitWriter) reset(writer io.Writer) {
114+ w.writer = writer
115+ w.bits, w.nbits, w.nbytes, w.err = 0, 0, 0, nil
116+ w.bytes = [bufferSize]byte{}
117+}
118+
119+func (w *huffmanBitWriter) flush() {
120+ if w.err != nil {
121+ w.nbits = 0
122+ return
123+ }
124+ n := w.nbytes
125+ for w.nbits != 0 {
126+ w.bytes[n] = byte(w.bits)
127+ w.bits >>= 8
128+ if w.nbits > 8 { // Avoid underflow
129+ w.nbits -= 8
130+ } else {
131+ w.nbits = 0
132+ }
133+ n++
134+ }
135+ w.bits = 0
136+ w.write(w.bytes[:n])
137+ w.nbytes = 0
138+}
139+
140+func (w *huffmanBitWriter) write(b []byte) {
141+ if w.err != nil {
142+ return
143+ }
144+ _, w.err = w.writer.Write(b)
145+}
146+
147+func (w *huffmanBitWriter) writeBits(b int32, nb uint) {
148+ if w.err != nil {
149+ return
150+ }
151+ w.bits |= uint64(b) << w.nbits
152+ w.nbits += nb
153+ if w.nbits >= 48 {
154+ bits := w.bits
155+ w.bits >>= 48
156+ w.nbits -= 48
157+ n := w.nbytes
158+ bytes := w.bytes[n : n+6]
159+ bytes[0] = byte(bits)
160+ bytes[1] = byte(bits >> 8)
161+ bytes[2] = byte(bits >> 16)
162+ bytes[3] = byte(bits >> 24)
163+ bytes[4] = byte(bits >> 32)
164+ bytes[5] = byte(bits >> 40)
165+ n += 6
166+ if n >= bufferFlushSize {
167+ w.write(w.bytes[:n])
168+ n = 0
169+ }
170+ w.nbytes = n
171+ }
172+}
173+
174+func (w *huffmanBitWriter) writeBytes(bytes []byte) {
175+ if w.err != nil {
176+ return
177+ }
178+ n := w.nbytes
179+ if w.nbits&7 != 0 {
180+ w.err = InternalError("writeBytes with unfinished bits")
181+ return
182+ }
183+ for w.nbits != 0 {
184+ w.bytes[n] = byte(w.bits)
185+ w.bits >>= 8
186+ w.nbits -= 8
187+ n++
188+ }
189+ if n != 0 {
190+ w.write(w.bytes[:n])
191+ }
192+ w.nbytes = 0
193+ w.write(bytes)
194+}
195+
196+// RFC 1951 3.2.7 specifies a special run-length encoding for specifying
197+// the literal and offset lengths arrays (which are concatenated into a single
198+// array). This method generates that run-length encoding.
199+//
200+// The result is written into the codegen array, and the frequencies
201+// of each code is written into the codegenFreq array.
202+// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional
203+// information. Code badCode is an end marker
204+//
205+// numLiterals The number of literals in literalEncoding
206+// numOffsets The number of offsets in offsetEncoding
207+// litenc, offenc The literal and offset encoder to use
208+func (w *huffmanBitWriter) generateCodegen(numLiterals int, numOffsets int, litEnc, offEnc *huffmanEncoder) {
209+ for i := range w.codegenFreq {
210+ w.codegenFreq[i] = 0
211+ }
212+ // Note that we are using codegen both as a temporary variable for holding
213+ // a copy of the frequencies, and as the place where we put the result.
214+ // This is fine because the output is always shorter than the input used
215+ // so far.
216+ codegen := w.codegen // cache
217+ // Copy the concatenated code sizes to codegen. Put a marker at the end.
218+ cgnl := codegen[:numLiterals]
219+ for i := range cgnl {
220+ cgnl[i] = uint8(litEnc.codes[i].len)
221+ }
222+
223+ cgnl = codegen[numLiterals : numLiterals+numOffsets]
224+ for i := range cgnl {
225+ cgnl[i] = uint8(offEnc.codes[i].len)
226+ }
227+ codegen[numLiterals+numOffsets] = badCode
228+
229+ size := codegen[0]
230+ count := 1
231+ outIndex := 0
232+ for inIndex := 1; size != badCode; inIndex++ {
233+ // INVARIANT: We have seen "count" copies of size that have not yet
234+ // had output generated for them.
235+ nextSize := codegen[inIndex]
236+ if nextSize == size {
237+ count++
238+ continue
239+ }
240+ // We need to generate codegen indicating "count" of size.
241+ if size != 0 {
242+ codegen[outIndex] = size
243+ outIndex++
244+ w.codegenFreq[size]++
245+ count--
246+ for count >= 3 {
247+ n := 6
248+ if n > count {
249+ n = count
250+ }
251+ codegen[outIndex] = 16
252+ outIndex++
253+ codegen[outIndex] = uint8(n - 3)
254+ outIndex++
255+ w.codegenFreq[16]++
256+ count -= n
257+ }
258+ } else {
259+ for count >= 11 {
260+ n := 138
261+ if n > count {
262+ n = count
263+ }
264+ codegen[outIndex] = 18
265+ outIndex++
266+ codegen[outIndex] = uint8(n - 11)
267+ outIndex++
268+ w.codegenFreq[18]++
269+ count -= n
270+ }
271+ if count >= 3 {
272+ // count >= 3 && count <= 10
273+ codegen[outIndex] = 17
274+ outIndex++
275+ codegen[outIndex] = uint8(count - 3)
276+ outIndex++
277+ w.codegenFreq[17]++
278+ count = 0
279+ }
280+ }
281+ count--
282+ for ; count >= 0; count-- {
283+ codegen[outIndex] = size
284+ outIndex++
285+ w.codegenFreq[size]++
286+ }
287+ // Set up invariant for next time through the loop.
288+ size = nextSize
289+ count = 1
290+ }
291+ // Marker indicating the end of the codegen.
292+ codegen[outIndex] = badCode
293+}
294+
295+// dynamicSize returns the size of dynamically encoded data in bits.
296+func (w *huffmanBitWriter) dynamicSize(litEnc, offEnc *huffmanEncoder, extraBits int) (size, numCodegens int) {
297+ numCodegens = len(w.codegenFreq)
298+ for numCodegens > 4 && w.codegenFreq[codegenOrder[numCodegens-1]] == 0 {
299+ numCodegens--
300+ }
301+ header := 3 + 5 + 5 + 4 + (3 * numCodegens) +
302+ w.codegenEncoding.bitLength(w.codegenFreq[:]) +
303+ int(w.codegenFreq[16])*2 +
304+ int(w.codegenFreq[17])*3 +
305+ int(w.codegenFreq[18])*7
306+ size = header +
307+ litEnc.bitLength(w.literalFreq) +
308+ offEnc.bitLength(w.offsetFreq) +
309+ extraBits
310+
311+ return size, numCodegens
312+}
313+
314+// fixedSize returns the size of dynamically encoded data in bits.
315+func (w *huffmanBitWriter) fixedSize(extraBits int) int {
316+ return 3 +
317+ fixedLiteralEncoding.bitLength(w.literalFreq) +
318+ fixedOffsetEncoding.bitLength(w.offsetFreq) +
319+ extraBits
320+}
321+
322+// storedSize calculates the stored size, including header.
323+// The function returns the size in bits and whether the block
324+// fits inside a single block.
325+func (w *huffmanBitWriter) storedSize(in []byte) (int, bool) {
326+ if in == nil {
327+ return 0, false
328+ }
329+ if len(in) <= maxStoreBlockSize {
330+ return (len(in) + 5) * 8, true
331+ }
332+ return 0, false
333+}
334+
335+func (w *huffmanBitWriter) writeCode(c hcode) {
336+ if w.err != nil {
337+ return
338+ }
339+ w.bits |= uint64(c.code) << w.nbits
340+ w.nbits += uint(c.len)
341+ if w.nbits >= 48 {
342+ bits := w.bits
343+ w.bits >>= 48
344+ w.nbits -= 48
345+ n := w.nbytes
346+ bytes := w.bytes[n : n+6]
347+ bytes[0] = byte(bits)
348+ bytes[1] = byte(bits >> 8)
349+ bytes[2] = byte(bits >> 16)
350+ bytes[3] = byte(bits >> 24)
351+ bytes[4] = byte(bits >> 32)
352+ bytes[5] = byte(bits >> 40)
353+ n += 6
354+ if n >= bufferFlushSize {
355+ w.write(w.bytes[:n])
356+ n = 0
357+ }
358+ w.nbytes = n
359+ }
360+}
361+
362+// Write the header of a dynamic Huffman block to the output stream.
363+//
364+// numLiterals The number of literals specified in codegen
365+// numOffsets The number of offsets specified in codegen
366+// numCodegens The number of codegens used in codegen
367+func (w *huffmanBitWriter) writeDynamicHeader(numLiterals int, numOffsets int, numCodegens int, isEof bool) {
368+ if w.err != nil {
369+ return
370+ }
371+ var firstBits int32 = 4
372+ if isEof {
373+ firstBits = 5
374+ }
375+ w.writeBits(firstBits, 3)
376+ w.writeBits(int32(numLiterals-257), 5)
377+ w.writeBits(int32(numOffsets-1), 5)
378+ w.writeBits(int32(numCodegens-4), 4)
379+
380+ for i := 0; i < numCodegens; i++ {
381+ value := uint(w.codegenEncoding.codes[codegenOrder[i]].len)
382+ w.writeBits(int32(value), 3)
383+ }
384+
385+ i := 0
386+ for {
387+ var codeWord int = int(w.codegen[i])
388+ i++
389+ if codeWord == badCode {
390+ break
391+ }
392+ w.writeCode(w.codegenEncoding.codes[uint32(codeWord)])
393+
394+ switch codeWord {
395+ case 16:
396+ w.writeBits(int32(w.codegen[i]), 2)
397+ i++
398+ break
399+ case 17:
400+ w.writeBits(int32(w.codegen[i]), 3)
401+ i++
402+ break
403+ case 18:
404+ w.writeBits(int32(w.codegen[i]), 7)
405+ i++
406+ break
407+ }
408+ }
409+}
410+
411+func (w *huffmanBitWriter) writeStoredHeader(length int, isEof bool) {
412+ if w.err != nil {
413+ return
414+ }
415+ var flag int32
416+ if isEof {
417+ flag = 1
418+ }
419+ w.writeBits(flag, 3)
420+ w.flush()
421+ w.writeBits(int32(length), 16)
422+ w.writeBits(int32(^uint16(length)), 16)
423+}
424+
425+func (w *huffmanBitWriter) writeFixedHeader(isEof bool) {
426+ if w.err != nil {
427+ return
428+ }
429+ // Indicate that we are a fixed Huffman block
430+ var value int32 = 2
431+ if isEof {
432+ value = 3
433+ }
434+ w.writeBits(value, 3)
435+}
436+
437+// writeBlock will write a block of tokens with the smallest encoding.
438+// The original input can be supplied, and if the huffman encoded data
439+// is larger than the original bytes, the data will be written as a
440+// stored block.
441+// If the input is nil, the tokens will always be Huffman encoded.
442+func (w *huffmanBitWriter) writeBlock(tokens []token, eof bool, input []byte) {
443+ if w.err != nil {
444+ return
445+ }
446+
447+ tokens = append(tokens, endBlockMarker)
448+ numLiterals, numOffsets := w.indexTokens(tokens)
449+
450+ var extraBits int
451+ storedSize, storable := w.storedSize(input)
452+ if storable {
453+ // We only bother calculating the costs of the extra bits required by
454+ // the length of offset fields (which will be the same for both fixed
455+ // and dynamic encoding), if we need to compare those two encodings
456+ // against stored encoding.
457+ for lengthCode := lengthCodesStart + 8; lengthCode < numLiterals; lengthCode++ {
458+ // First eight length codes have extra size = 0.
459+ extraBits += int(w.literalFreq[lengthCode]) * int(lengthExtraBits[lengthCode-lengthCodesStart])
460+ }
461+ for offsetCode := 4; offsetCode < numOffsets; offsetCode++ {
462+ // First four offset codes have extra size = 0.
463+ extraBits += int(w.offsetFreq[offsetCode]) * int(offsetExtraBits[offsetCode])
464+ }
465+ }
466+
467+ // Figure out smallest code.
468+ // Fixed Huffman baseline.
469+ var literalEncoding = fixedLiteralEncoding
470+ var offsetEncoding = fixedOffsetEncoding
471+ var size = w.fixedSize(extraBits)
472+
473+ // Dynamic Huffman?
474+ var numCodegens int
475+
476+ // Generate codegen and codegenFrequencies, which indicates how to encode
477+ // the literalEncoding and the offsetEncoding.
478+ w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding)
479+ w.codegenEncoding.generate(w.codegenFreq[:], 7)
480+ dynamicSize, numCodegens := w.dynamicSize(w.literalEncoding, w.offsetEncoding, extraBits)
481+
482+ if dynamicSize < size {
483+ size = dynamicSize
484+ literalEncoding = w.literalEncoding
485+ offsetEncoding = w.offsetEncoding
486+ }
487+
488+ // Stored bytes?
489+ if storable && storedSize < size {
490+ w.writeStoredHeader(len(input), eof)
491+ w.writeBytes(input)
492+ return
493+ }
494+
495+ // Huffman.
496+ if literalEncoding == fixedLiteralEncoding {
497+ w.writeFixedHeader(eof)
498+ } else {
499+ w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof)
500+ }
501+
502+ // Write the tokens.
503+ w.writeTokens(tokens, literalEncoding.codes, offsetEncoding.codes)
504+}
505+
506+// writeBlockDynamic encodes a block using a dynamic Huffman table.
507+// This should be used if the symbols used have a disproportionate
508+// histogram distribution.
509+// If input is supplied and the compression savings are below 1/16th of the
510+// input size the block is stored.
511+func (w *huffmanBitWriter) writeBlockDynamic(tokens []token, eof bool, input []byte) {
512+ if w.err != nil {
513+ return
514+ }
515+
516+ tokens = append(tokens, endBlockMarker)
517+ numLiterals, numOffsets := w.indexTokens(tokens)
518+
519+ // Generate codegen and codegenFrequencies, which indicates how to encode
520+ // the literalEncoding and the offsetEncoding.
521+ w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding)
522+ w.codegenEncoding.generate(w.codegenFreq[:], 7)
523+ size, numCodegens := w.dynamicSize(w.literalEncoding, w.offsetEncoding, 0)
524+
525+ // Store bytes, if we don't get a reasonable improvement.
526+ if ssize, storable := w.storedSize(input); storable && ssize < (size+size>>4) {
527+ w.writeStoredHeader(len(input), eof)
528+ w.writeBytes(input)
529+ return
530+ }
531+
532+ // Write Huffman table.
533+ w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof)
534+
535+ // Write the tokens.
536+ w.writeTokens(tokens, w.literalEncoding.codes, w.offsetEncoding.codes)
537+}
538+
539+// indexTokens indexes a slice of tokens, and updates
540+// literalFreq and offsetFreq, and generates literalEncoding
541+// and offsetEncoding.
542+// The number of literal and offset tokens is returned.
543+func (w *huffmanBitWriter) indexTokens(tokens []token) (numLiterals, numOffsets int) {
544+ for i := range w.literalFreq {
545+ w.literalFreq[i] = 0
546+ }
547+ for i := range w.offsetFreq {
548+ w.offsetFreq[i] = 0
549+ }
550+
551+ for _, t := range tokens {
552+ if t < matchType {
553+ w.literalFreq[t.literal()]++
554+ continue
555+ }
556+ length := t.length()
557+ offset := t.offset()
558+ w.literalFreq[lengthCodesStart+lengthCode(length)]++
559+ w.offsetFreq[offsetCode(offset)]++
560+ }
561+
562+ // get the number of literals
563+ numLiterals = len(w.literalFreq)
564+ for w.literalFreq[numLiterals-1] == 0 {
565+ numLiterals--
566+ }
567+ // get the number of offsets
568+ numOffsets = len(w.offsetFreq)
569+ for numOffsets > 0 && w.offsetFreq[numOffsets-1] == 0 {
570+ numOffsets--
571+ }
572+ if numOffsets == 0 {
573+ // We haven't found a single match. If we want to go with the dynamic encoding,
574+ // we should count at least one offset to be sure that the offset huffman tree could be encoded.
575+ w.offsetFreq[0] = 1
576+ numOffsets = 1
577+ }
578+ w.literalEncoding.generate(w.literalFreq, 15)
579+ w.offsetEncoding.generate(w.offsetFreq, 15)
580+ return
581+}
582+
583+// writeTokens writes a slice of tokens to the output.
584+// codes for literal and offset encoding must be supplied.
585+func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode) {
586+ if w.err != nil {
587+ return
588+ }
589+ for _, t := range tokens {
590+ if t < matchType {
591+ w.writeCode(leCodes[t.literal()])
592+ continue
593+ }
594+ // Write the length
595+ length := t.length()
596+ lengthCode := lengthCode(length)
597+ w.writeCode(leCodes[lengthCode+lengthCodesStart])
598+ extraLengthBits := uint(lengthExtraBits[lengthCode])
599+ if extraLengthBits > 0 {
600+ extraLength := int32(length - lengthBase[lengthCode])
601+ w.writeBits(extraLength, extraLengthBits)
602+ }
603+ // Write the offset
604+ offset := t.offset()
605+ offsetCode := offsetCode(offset)
606+ w.writeCode(oeCodes[offsetCode])
607+ extraOffsetBits := uint(offsetExtraBits[offsetCode])
608+ if extraOffsetBits > 0 {
609+ extraOffset := int32(offset - offsetBase[offsetCode])
610+ w.writeBits(extraOffset, extraOffsetBits)
611+ }
612+ }
613+}
614+
615+// huffOffset is a static offset encoder used for huffman only encoding.
616+// It can be reused since we will not be encoding offset values.
617+var huffOffset *huffmanEncoder
618+
619+func init() {
620+ w := newHuffmanBitWriter(nil)
621+ w.offsetFreq[0] = 1
622+ huffOffset = newHuffmanEncoder(offsetCodeCount)
623+ huffOffset.generate(w.offsetFreq, 15)
624+}
625+
626+// writeBlockHuff encodes a block of bytes as either
627+// Huffman encoded literals or uncompressed bytes if the
628+// results only gains very little from compression.
629+func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte) {
630+ if w.err != nil {
631+ return
632+ }
633+
634+ // Clear histogram
635+ for i := range w.literalFreq {
636+ w.literalFreq[i] = 0
637+ }
638+
639+ // Add everything as literals
640+ histogram(input, w.literalFreq)
641+
642+ w.literalFreq[endBlockMarker] = 1
643+
644+ const numLiterals = endBlockMarker + 1
645+ const numOffsets = 1
646+
647+ w.literalEncoding.generate(w.literalFreq, 15)
648+
649+ // Figure out smallest code.
650+ // Always use dynamic Huffman or Store
651+ var numCodegens int
652+
653+ // Generate codegen and codegenFrequencies, which indicates how to encode
654+ // the literalEncoding and the offsetEncoding.
655+ w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, huffOffset)
656+ w.codegenEncoding.generate(w.codegenFreq[:], 7)
657+ size, numCodegens := w.dynamicSize(w.literalEncoding, huffOffset, 0)
658+
659+ // Store bytes, if we don't get a reasonable improvement.
660+ if ssize, storable := w.storedSize(input); storable && ssize < (size+size>>4) {
661+ w.writeStoredHeader(len(input), eof)
662+ w.writeBytes(input)
663+ return
664+ }
665+
666+ // Huffman.
667+ w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof)
668+ encoding := w.literalEncoding.codes[:257]
669+ n := w.nbytes
670+ for _, t := range input {
671+ // Bitwriting inlined, ~30% speedup
672+ c := encoding[t]
673+ w.bits |= uint64(c.code) << w.nbits
674+ w.nbits += uint(c.len)
675+ if w.nbits < 48 {
676+ continue
677+ }
678+ // Store 6 bytes
679+ bits := w.bits
680+ w.bits >>= 48
681+ w.nbits -= 48
682+ bytes := w.bytes[n : n+6]
683+ bytes[0] = byte(bits)
684+ bytes[1] = byte(bits >> 8)
685+ bytes[2] = byte(bits >> 16)
686+ bytes[3] = byte(bits >> 24)
687+ bytes[4] = byte(bits >> 32)
688+ bytes[5] = byte(bits >> 40)
689+ n += 6
690+ if n < bufferFlushSize {
691+ continue
692+ }
693+ w.write(w.bytes[:n])
694+ if w.err != nil {
695+ return // Return early in the event of write failures
696+ }
697+ n = 0
698+ }
699+ w.nbytes = n
700+ w.writeCode(encoding[endBlockMarker])
701+}
vendor/github.com/klauspost/compress/flate/huffman_code.go+344−0
@@ -0,0 +1,344 @@
1+// Copyright 2009 The Go Authors. All rights reserved.
2+// Use of this source code is governed by a BSD-style
3+// license that can be found in the LICENSE file.
4+
5+package flate
6+
7+import (
8+ "math"
9+ "sort"
10+)
11+
12+// hcode is a huffman code with a bit code and bit length.
13+type hcode struct {
14+ code, len uint16
15+}
16+
17+type huffmanEncoder struct {
18+ codes []hcode
19+ freqcache []literalNode
20+ bitCount [17]int32
21+ lns byLiteral // stored to avoid repeated allocation in generate
22+ lfs byFreq // stored to avoid repeated allocation in generate
23+}
24+
25+type literalNode struct {
26+ literal uint16
27+ freq int32
28+}
29+
30+// A levelInfo describes the state of the constructed tree for a given depth.
31+type levelInfo struct {
32+ // Our level. for better printing
33+ level int32
34+
35+ // The frequency of the last node at this level
36+ lastFreq int32
37+
38+ // The frequency of the next character to add to this level
39+ nextCharFreq int32
40+
41+ // The frequency of the next pair (from level below) to add to this level.
42+ // Only valid if the "needed" value of the next lower level is 0.
43+ nextPairFreq int32
44+
45+ // The number of chains remaining to generate for this level before moving
46+ // up to the next level
47+ needed int32
48+}
49+
50+// set sets the code and length of an hcode.
51+func (h *hcode) set(code uint16, length uint16) {
52+ h.len = length
53+ h.code = code
54+}
55+
56+func maxNode() literalNode { return literalNode{math.MaxUint16, math.MaxInt32} }
57+
58+func newHuffmanEncoder(size int) *huffmanEncoder {
59+ return &huffmanEncoder{codes: make([]hcode, size)}
60+}
61+
62+// Generates a HuffmanCode corresponding to the fixed literal table
63+func generateFixedLiteralEncoding() *huffmanEncoder {
64+ h := newHuffmanEncoder(maxNumLit)
65+ codes := h.codes
66+ var ch uint16
67+ for ch = 0; ch < maxNumLit; ch++ {
68+ var bits uint16
69+ var size uint16
70+ switch {
71+ case ch < 144:
72+ // size 8, 000110000 .. 10111111
73+ bits = ch + 48
74+ size = 8
75+ break
76+ case ch < 256:
77+ // size 9, 110010000 .. 111111111
78+ bits = ch + 400 - 144
79+ size = 9
80+ break
81+ case ch < 280:
82+ // size 7, 0000000 .. 0010111
83+ bits = ch - 256
84+ size = 7
85+ break
86+ default:
87+ // size 8, 11000000 .. 11000111
88+ bits = ch + 192 - 280
89+ size = 8
90+ }
91+ codes[ch] = hcode{code: reverseBits(bits, byte(size)), len: size}
92+ }
93+ return h
94+}
95+
96+func generateFixedOffsetEncoding() *huffmanEncoder {
97+ h := newHuffmanEncoder(30)
98+ codes := h.codes
99+ for ch := range codes {
100+ codes[ch] = hcode{code: reverseBits(uint16(ch), 5), len: 5}
101+ }
102+ return h
103+}
104+
105+var fixedLiteralEncoding *huffmanEncoder = generateFixedLiteralEncoding()
106+var fixedOffsetEncoding *huffmanEncoder = generateFixedOffsetEncoding()
107+
108+func (h *huffmanEncoder) bitLength(freq []int32) int {
109+ var total int
110+ for i, f := range freq {
111+ if f != 0 {
112+ total += int(f) * int(h.codes[i].len)
113+ }
114+ }
115+ return total
116+}
117+
118+const maxBitsLimit = 16
119+
120+// Return the number of literals assigned to each bit size in the Huffman encoding
121+//
122+// This method is only called when list.length >= 3
123+// The cases of 0, 1, and 2 literals are handled by special case code.
124+//
125+// list An array of the literals with non-zero frequencies
126+// and their associated frequencies. The array is in order of increasing
127+// frequency, and has as its last element a special element with frequency
128+// MaxInt32
129+// maxBits The maximum number of bits that should be used to encode any literal.
130+// Must be less than 16.
131+// return An integer array in which array[i] indicates the number of literals
132+// that should be encoded in i bits.
133+func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 {
134+ if maxBits >= maxBitsLimit {
135+ panic("flate: maxBits too large")
136+ }
137+ n := int32(len(list))
138+ list = list[0 : n+1]
139+ list[n] = maxNode()
140+
141+ // The tree can't have greater depth than n - 1, no matter what. This
142+ // saves a little bit of work in some small cases
143+ if maxBits > n-1 {
144+ maxBits = n - 1
145+ }
146+
147+ // Create information about each of the levels.
148+ // A bogus "Level 0" whose sole purpose is so that
149+ // level1.prev.needed==0. This makes level1.nextPairFreq
150+ // be a legitimate value that never gets chosen.
151+ var levels [maxBitsLimit]levelInfo
152+ // leafCounts[i] counts the number of literals at the left
153+ // of ancestors of the rightmost node at level i.
154+ // leafCounts[i][j] is the number of literals at the left
155+ // of the level j ancestor.
156+ var leafCounts [maxBitsLimit][maxBitsLimit]int32
157+
158+ for level := int32(1); level <= maxBits; level++ {
159+ // For every level, the first two items are the first two characters.
160+ // We initialize the levels as if we had already figured this out.
161+ levels[level] = levelInfo{
162+ level: level,
163+ lastFreq: list[1].freq,
164+ nextCharFreq: list[2].freq,
165+ nextPairFreq: list[0].freq + list[1].freq,
166+ }
167+ leafCounts[level][level] = 2
168+ if level == 1 {
169+ levels[level].nextPairFreq = math.MaxInt32
170+ }
171+ }
172+
173+ // We need a total of 2*n - 2 items at top level and have already generated 2.
174+ levels[maxBits].needed = 2*n - 4
175+
176+ level := maxBits
177+ for {
178+ l := &levels[level]
179+ if l.nextPairFreq == math.MaxInt32 && l.nextCharFreq == math.MaxInt32 {
180+ // We've run out of both leafs and pairs.
181+ // End all calculations for this level.
182+ // To make sure we never come back to this level or any lower level,
183+ // set nextPairFreq impossibly large.
184+ l.needed = 0
185+ levels[level+1].nextPairFreq = math.MaxInt32
186+ level++
187+ continue
188+ }
189+
190+ prevFreq := l.lastFreq
191+ if l.nextCharFreq < l.nextPairFreq {
192+ // The next item on this row is a leaf node.
193+ n := leafCounts[level][level] + 1
194+ l.lastFreq = l.nextCharFreq
195+ // Lower leafCounts are the same of the previous node.
196+ leafCounts[level][level] = n
197+ l.nextCharFreq = list[n].freq
198+ } else {
199+ // The next item on this row is a pair from the previous row.
200+ // nextPairFreq isn't valid until we generate two
201+ // more values in the level below
202+ l.lastFreq = l.nextPairFreq
203+ // Take leaf counts from the lower level, except counts[level] remains the same.
204+ copy(leafCounts[level][:level], leafCounts[level-1][:level])
205+ levels[l.level-1].needed = 2
206+ }
207+
208+ if l.needed--; l.needed == 0 {
209+ // We've done everything we need to do for this level.
210+ // Continue calculating one level up. Fill in nextPairFreq
211+ // of that level with the sum of the two nodes we've just calculated on
212+ // this level.
213+ if l.level == maxBits {
214+ // All done!
215+ break
216+ }
217+ levels[l.level+1].nextPairFreq = prevFreq + l.lastFreq
218+ level++
219+ } else {
220+ // If we stole from below, move down temporarily to replenish it.
221+ for levels[level-1].needed > 0 {
222+ level--
223+ }
224+ }
225+ }
226+
227+ // Somethings is wrong if at the end, the top level is null or hasn't used
228+ // all of the leaves.
229+ if leafCounts[maxBits][maxBits] != n {
230+ panic("leafCounts[maxBits][maxBits] != n")
231+ }
232+
233+ bitCount := h.bitCount[:maxBits+1]
234+ bits := 1
235+ counts := &leafCounts[maxBits]
236+ for level := maxBits; level > 0; level-- {
237+ // chain.leafCount gives the number of literals requiring at least "bits"
238+ // bits to encode.
239+ bitCount[bits] = counts[level] - counts[level-1]
240+ bits++
241+ }
242+ return bitCount
243+}
244+
245+// Look at the leaves and assign them a bit count and an encoding as specified
246+// in RFC 1951 3.2.2
247+func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode) {
248+ code := uint16(0)
249+ for n, bits := range bitCount {
250+ code <<= 1
251+ if n == 0 || bits == 0 {
252+ continue
253+ }
254+ // The literals list[len(list)-bits] .. list[len(list)-bits]
255+ // are encoded using "bits" bits, and get the values
256+ // code, code + 1, .... The code values are
257+ // assigned in literal order (not frequency order).
258+ chunk := list[len(list)-int(bits):]
259+
260+ h.lns.sort(chunk)
261+ for _, node := range chunk {
262+ h.codes[node.literal] = hcode{code: reverseBits(code, uint8(n)), len: uint16(n)}
263+ code++
264+ }
265+ list = list[0 : len(list)-int(bits)]
266+ }
267+}
268+
269+// Update this Huffman Code object to be the minimum code for the specified frequency count.
270+//
271+// freq An array of frequencies, in which frequency[i] gives the frequency of literal i.
272+// maxBits The maximum number of bits to use for any literal.
273+func (h *huffmanEncoder) generate(freq []int32, maxBits int32) {
274+ if h.freqcache == nil {
275+ // Allocate a reusable buffer with the longest possible frequency table.
276+ // Possible lengths are codegenCodeCount, offsetCodeCount and maxNumLit.
277+ // The largest of these is maxNumLit, so we allocate for that case.
278+ h.freqcache = make([]literalNode, maxNumLit+1)
279+ }
280+ list := h.freqcache[:len(freq)+1]
281+ // Number of non-zero literals
282+ count := 0
283+ // Set list to be the set of all non-zero literals and their frequencies
284+ for i, f := range freq {
285+ if f != 0 {
286+ list[count] = literalNode{uint16(i), f}
287+ count++
288+ } else {
289+ list[count] = literalNode{}
290+ h.codes[i].len = 0
291+ }
292+ }
293+ list[len(freq)] = literalNode{}
294+
295+ list = list[:count]
296+ if count <= 2 {
297+ // Handle the small cases here, because they are awkward for the general case code. With
298+ // two or fewer literals, everything has bit length 1.
299+ for i, node := range list {
300+ // "list" is in order of increasing literal value.
301+ h.codes[node.literal].set(uint16(i), 1)
302+ }
303+ return
304+ }
305+ h.lfs.sort(list)
306+
307+ // Get the number of literals for each bit count
308+ bitCount := h.bitCounts(list, maxBits)
309+ // And do the assignment
310+ h.assignEncodingAndSize(bitCount, list)
311+}
312+
313+type byLiteral []literalNode
314+
315+func (s *byLiteral) sort(a []literalNode) {
316+ *s = byLiteral(a)
317+ sort.Sort(s)
318+}
319+
320+func (s byLiteral) Len() int { return len(s) }
321+
322+func (s byLiteral) Less(i, j int) bool {
323+ return s[i].literal < s[j].literal
324+}
325+
326+func (s byLiteral) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
327+
328+type byFreq []literalNode
329+
330+func (s *byFreq) sort(a []literalNode) {
331+ *s = byFreq(a)
332+ sort.Sort(s)
333+}
334+
335+func (s byFreq) Len() int { return len(s) }
336+
337+func (s byFreq) Less(i, j int) bool {
338+ if s[i].freq == s[j].freq {
339+ return s[i].literal < s[j].literal
340+ }
341+ return s[i].freq < s[j].freq
342+}
343+
344+func (s byFreq) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
vendor/github.com/klauspost/compress/flate/inflate.go+868−0
@@ -0,0 +1,868 @@
1+// Copyright 2009 The Go Authors. All rights reserved.
2+// Use of this source code is governed by a BSD-style
3+// license that can be found in the LICENSE file.
4+
5+// Package flate implements the DEFLATE compressed data format, described in
6+// RFC 1951. The gzip and zlib packages implement access to DEFLATE-based file
7+// formats.
8+package flate
9+
10+import (
11+ "bufio"
12+ "io"
13+ "strconv"
14+ "sync"
15+)
16+
17+const (
18+ maxCodeLen = 16 // max length of Huffman code
19+ maxCodeLenMask = 15 // mask for max length of Huffman code
20+ // The next three numbers come from the RFC section 3.2.7, with the
21+ // additional proviso in section 3.2.5 which implies that distance codes
22+ // 30 and 31 should never occur in compressed data.
23+ maxNumLit = 286
24+ maxNumDist = 30
25+ numCodes = 19 // number of codes in Huffman meta-code
26+)
27+
28+// Initialize the fixedHuffmanDecoder only once upon first use.
29+var fixedOnce sync.Once
30+var fixedHuffmanDecoder huffmanDecoder
31+
32+// A CorruptInputError reports the presence of corrupt input at a given offset.
33+type CorruptInputError int64
34+
35+func (e CorruptInputError) Error() string {
36+ return "flate: corrupt input before offset " + strconv.FormatInt(int64(e), 10)
37+}
38+
39+// An InternalError reports an error in the flate code itself.
40+type InternalError string
41+
42+func (e InternalError) Error() string { return "flate: internal error: " + string(e) }
43+
44+// A ReadError reports an error encountered while reading input.
45+//
46+// Deprecated: No longer returned.
47+type ReadError struct {
48+ Offset int64 // byte offset where error occurred
49+ Err error // error returned by underlying Read
50+}
51+
52+func (e *ReadError) Error() string {
53+ return "flate: read error at offset " + strconv.FormatInt(e.Offset, 10) + ": " + e.Err.Error()
54+}
55+
56+// A WriteError reports an error encountered while writing output.
57+//
58+// Deprecated: No longer returned.
59+type WriteError struct {
60+ Offset int64 // byte offset where error occurred
61+ Err error // error returned by underlying Write
62+}
63+
64+func (e *WriteError) Error() string {
65+ return "flate: write error at offset " + strconv.FormatInt(e.Offset, 10) + ": " + e.Err.Error()
66+}
67+
68+// Resetter resets a ReadCloser returned by NewReader or NewReaderDict to
69+// to switch to a new underlying Reader. This permits reusing a ReadCloser
70+// instead of allocating a new one.
71+type Resetter interface {
72+ // Reset discards any buffered data and resets the Resetter as if it was
73+ // newly initialized with the given reader.
74+ Reset(r io.Reader, dict []byte) error
75+}
76+
77+// The data structure for decoding Huffman tables is based on that of
78+// zlib. There is a lookup table of a fixed bit width (huffmanChunkBits),
79+// For codes smaller than the table width, there are multiple entries
80+// (each combination of trailing bits has the same value). For codes
81+// larger than the table width, the table contains a link to an overflow
82+// table. The width of each entry in the link table is the maximum code
83+// size minus the chunk width.
84+//
85+// Note that you can do a lookup in the table even without all bits
86+// filled. Since the extra bits are zero, and the DEFLATE Huffman codes
87+// have the property that shorter codes come before longer ones, the
88+// bit length estimate in the result is a lower bound on the actual
89+// number of bits.
90+//
91+// See the following:
92+// http://www.gzip.org/algorithm.txt
93+
94+// chunk & 15 is number of bits
95+// chunk >> 4 is value, including table link
96+
97+const (
98+ huffmanChunkBits = 9
99+ huffmanNumChunks = 1 << huffmanChunkBits
100+ huffmanCountMask = 15
101+ huffmanValueShift = 4
102+)
103+
104+type huffmanDecoder struct {
105+ min int // the minimum code length
106+ chunks *[huffmanNumChunks]uint32 // chunks as described above
107+ links [][]uint32 // overflow links
108+ linkMask uint32 // mask the width of the link table
109+}
110+
111+// Initialize Huffman decoding tables from array of code lengths.
112+// Following this function, h is guaranteed to be initialized into a complete
113+// tree (i.e., neither over-subscribed nor under-subscribed). The exception is a
114+// degenerate case where the tree has only a single symbol with length 1. Empty
115+// trees are permitted.
116+func (h *huffmanDecoder) init(bits []int) bool {
117+ // Sanity enables additional runtime tests during Huffman
118+ // table construction. It's intended to be used during
119+ // development to supplement the currently ad-hoc unit tests.
120+ const sanity = false
121+
122+ if h.chunks == nil {
123+ h.chunks = &[huffmanNumChunks]uint32{}
124+ }
125+ if h.min != 0 {
126+ *h = huffmanDecoder{chunks: h.chunks, links: h.links}
127+ }
128+
129+ // Count number of codes of each length,
130+ // compute min and max length.
131+ var count [maxCodeLen]int
132+ var min, max int
133+ for _, n := range bits {
134+ if n == 0 {
135+ continue
136+ }
137+ if min == 0 || n < min {
138+ min = n
139+ }
140+ if n > max {
141+ max = n
142+ }
143+ count[n&maxCodeLenMask]++
144+ }
145+
146+ // Empty tree. The decompressor.huffSym function will fail later if the tree
147+ // is used. Technically, an empty tree is only valid for the HDIST tree and
148+ // not the HCLEN and HLIT tree. However, a stream with an empty HCLEN tree
149+ // is guaranteed to fail since it will attempt to use the tree to decode the
150+ // codes for the HLIT and HDIST trees. Similarly, an empty HLIT tree is
151+ // guaranteed to fail later since the compressed data section must be
152+ // composed of at least one symbol (the end-of-block marker).
153+ if max == 0 {
154+ return true
155+ }
156+
157+ code := 0
158+ var nextcode [maxCodeLen]int
159+ for i := min; i <= max; i++ {
160+ code <<= 1
161+ nextcode[i&maxCodeLenMask] = code
162+ code += count[i&maxCodeLenMask]
163+ }
164+
165+ // Check that the coding is complete (i.e., that we've
166+ // assigned all 2-to-the-max possible bit sequences).
167+ // Exception: To be compatible with zlib, we also need to
168+ // accept degenerate single-code codings. See also
169+ // TestDegenerateHuffmanCoding.
170+ if code != 1<<uint(max) && !(code == 1 && max == 1) {
171+ return false
172+ }
173+
174+ h.min = min
175+ chunks := h.chunks[:]
176+ for i := range chunks {
177+ chunks[i] = 0
178+ }
179+
180+ if max > huffmanChunkBits {
181+ numLinks := 1 << (uint(max) - huffmanChunkBits)
182+ h.linkMask = uint32(numLinks - 1)
183+
184+ // create link tables
185+ link := nextcode[huffmanChunkBits+1] >> 1
186+ if cap(h.links) < huffmanNumChunks-link {
187+ h.links = make([][]uint32, huffmanNumChunks-link)
188+ } else {
189+ h.links = h.links[:huffmanNumChunks-link]
190+ }
191+ for j := uint(link); j < huffmanNumChunks; j++ {
192+ reverse := int(reverseByte[j>>8]) | int(reverseByte[j&0xff])<<8
193+ reverse >>= uint(16 - huffmanChunkBits)
194+ off := j - uint(link)
195+ if sanity && h.chunks[reverse] != 0 {
196+ panic("impossible: overwriting existing chunk")
197+ }
198+ h.chunks[reverse] = uint32(off<<huffmanValueShift | (huffmanChunkBits + 1))
199+ if cap(h.links[off]) < numLinks {
200+ h.links[off] = make([]uint32, numLinks)
201+ } else {
202+ links := h.links[off][:0]
203+ h.links[off] = links[:numLinks]
204+ }
205+ }
206+ } else {
207+ h.links = h.links[:0]
208+ }
209+
210+ for i, n := range bits {
211+ if n == 0 {
212+ continue
213+ }
214+ code := nextcode[n]
215+ nextcode[n]++
216+ chunk := uint32(i<<huffmanValueShift | n)
217+ reverse := int(reverseByte[code>>8]) | int(reverseByte[code&0xff])<<8
218+ reverse >>= uint(16 - n)
219+ if n <= huffmanChunkBits {
220+ for off := reverse; off < len(h.chunks); off += 1 << uint(n) {
221+ // We should never need to overwrite
222+ // an existing chunk. Also, 0 is
223+ // never a valid chunk, because the
224+ // lower 4 "count" bits should be
225+ // between 1 and 15.
226+ if sanity && h.chunks[off] != 0 {
227+ panic("impossible: overwriting existing chunk")
228+ }
229+ h.chunks[off] = chunk
230+ }
231+ } else {
232+ j := reverse & (huffmanNumChunks - 1)
233+ if sanity && h.chunks[j]&huffmanCountMask != huffmanChunkBits+1 {
234+ // Longer codes should have been
235+ // associated with a link table above.
236+ panic("impossible: not an indirect chunk")
237+ }
238+ value := h.chunks[j] >> huffmanValueShift
239+ linktab := h.links[value]
240+ reverse >>= huffmanChunkBits
241+ for off := reverse; off < len(linktab); off += 1 << uint(n-huffmanChunkBits) {
242+ if sanity && linktab[off] != 0 {
243+ panic("impossible: overwriting existing chunk")
244+ }
245+ linktab[off] = chunk
246+ }
247+ }
248+ }
249+
250+ if sanity {
251+ // Above we've sanity checked that we never overwrote
252+ // an existing entry. Here we additionally check that
253+ // we filled the tables completely.
254+ for i, chunk := range h.chunks {
255+ if chunk == 0 {
256+ // As an exception, in the degenerate
257+ // single-code case, we allow odd
258+ // chunks to be missing.
259+ if code == 1 && i%2 == 1 {
260+ continue
261+ }
262+ panic("impossible: missing chunk")
263+ }
264+ }
265+ for _, linktab := range h.links {
266+ for _, chunk := range linktab {
267+ if chunk == 0 {
268+ panic("impossible: missing chunk")
269+ }
270+ }
271+ }
272+ }
273+
274+ return true
275+}
276+
277+// The actual read interface needed by NewReader.
278+// If the passed in io.Reader does not also have ReadByte,
279+// the NewReader will introduce its own buffering.
280+type Reader interface {
281+ io.Reader
282+ io.ByteReader
283+}
284+
285+// Decompress state.
286+type decompressor struct {
287+ // Input source.
288+ r Reader
289+ roffset int64
290+
291+ // Input bits, in top of b.
292+ b uint32
293+ nb uint
294+
295+ // Huffman decoders for literal/length, distance.
296+ h1, h2 huffmanDecoder
297+
298+ // Length arrays used to define Huffman codes.
299+ bits *[maxNumLit + maxNumDist]int
300+ codebits *[numCodes]int
301+
302+ // Output history, buffer.
303+ dict dictDecoder
304+
305+ // Temporary buffer (avoids repeated allocation).
306+ buf [4]byte
307+
308+ // Next step in the decompression,
309+ // and decompression state.
310+ step func(*decompressor)
311+ stepState int
312+ final bool
313+ err error
314+ toRead []byte
315+ hl, hd *huffmanDecoder
316+ copyLen int
317+ copyDist int
318+}
319+
320+func (f *decompressor) nextBlock() {
321+ for f.nb < 1+2 {
322+ if f.err = f.moreBits(); f.err != nil {
323+ return
324+ }
325+ }
326+ f.final = f.b&1 == 1
327+ f.b >>= 1
328+ typ := f.b & 3
329+ f.b >>= 2
330+ f.nb -= 1 + 2
331+ switch typ {
332+ case 0:
333+ f.dataBlock()
334+ case 1:
335+ // compressed, fixed Huffman tables
336+ f.hl = &fixedHuffmanDecoder
337+ f.hd = nil
338+ f.huffmanBlock()
339+ case 2:
340+ // compressed, dynamic Huffman tables
341+ if f.err = f.readHuffman(); f.err != nil {
342+ break
343+ }
344+ f.hl = &f.h1
345+ f.hd = &f.h2
346+ f.huffmanBlock()
347+ default:
348+ // 3 is reserved.
349+ f.err = CorruptInputError(f.roffset)
350+ }
351+}
352+
353+func (f *decompressor) Read(b []byte) (int, error) {
354+ for {
355+ if len(f.toRead) > 0 {
356+ n := copy(b, f.toRead)
357+ f.toRead = f.toRead[n:]
358+ if len(f.toRead) == 0 {
359+ return n, f.err
360+ }
361+ return n, nil
362+ }
363+ if f.err != nil {
364+ return 0, f.err
365+ }
366+ f.step(f)
367+ if f.err != nil && len(f.toRead) == 0 {
368+ f.toRead = f.dict.readFlush() // Flush what's left in case of error
369+ }
370+ }
371+}
372+
373+// Support the io.WriteTo interface for io.Copy and friends.
374+func (f *decompressor) WriteTo(w io.Writer) (int64, error) {
375+ total := int64(0)
376+ flushed := false
377+ for {
378+ if len(f.toRead) > 0 {
379+ n, err := w.Write(f.toRead)
380+ total += int64(n)
381+ if err != nil {
382+ f.err = err
383+ return total, err
384+ }
385+ if n != len(f.toRead) {
386+ return total, io.ErrShortWrite
387+ }
388+ f.toRead = f.toRead[:0]
389+ }
390+ if f.err != nil && flushed {
391+ if f.err == io.EOF {
392+ return total, nil
393+ }
394+ return total, f.err
395+ }
396+ if f.err == nil {
397+ f.step(f)
398+ }
399+ if len(f.toRead) == 0 && f.err != nil && !flushed {
400+ f.toRead = f.dict.readFlush() // Flush what's left in case of error
401+ flushed = true
402+ }
403+ }
404+}
405+
406+func (f *decompressor) Close() error {
407+ if f.err == io.EOF {
408+ return nil
409+ }
410+ return f.err
411+}
412+
413+// RFC 1951 section 3.2.7.
414+// Compression with dynamic Huffman codes
415+
416+var codeOrder = [...]int{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}
417+
418+func (f *decompressor) readHuffman() error {
419+ // HLIT[5], HDIST[5], HCLEN[4].
420+ for f.nb < 5+5+4 {
421+ if err := f.moreBits(); err != nil {
422+ return err
423+ }
424+ }
425+ nlit := int(f.b&0x1F) + 257
426+ if nlit > maxNumLit {
427+ return CorruptInputError(f.roffset)
428+ }
429+ f.b >>= 5
430+ ndist := int(f.b&0x1F) + 1
431+ if ndist > maxNumDist {
432+ return CorruptInputError(f.roffset)
433+ }
434+ f.b >>= 5
435+ nclen := int(f.b&0xF) + 4
436+ // numCodes is 19, so nclen is always valid.
437+ f.b >>= 4
438+ f.nb -= 5 + 5 + 4
439+
440+ // (HCLEN+4)*3 bits: code lengths in the magic codeOrder order.
441+ for i := 0; i < nclen; i++ {
442+ for f.nb < 3 {
443+ if err := f.moreBits(); err != nil {
444+ return err
445+ }
446+ }
447+ f.codebits[codeOrder[i]] = int(f.b & 0x7)
448+ f.b >>= 3
449+ f.nb -= 3
450+ }
451+ for i := nclen; i < len(codeOrder); i++ {
452+ f.codebits[codeOrder[i]] = 0
453+ }
454+ if !f.h1.init(f.codebits[0:]) {
455+ return CorruptInputError(f.roffset)
456+ }
457+
458+ // HLIT + 257 code lengths, HDIST + 1 code lengths,
459+ // using the code length Huffman code.
460+ for i, n := 0, nlit+ndist; i < n; {
461+ x, err := f.huffSym(&f.h1)
462+ if err != nil {
463+ return err
464+ }
465+ if x < 16 {
466+ // Actual length.
467+ f.bits[i] = x
468+ i++
469+ continue
470+ }
471+ // Repeat previous length or zero.
472+ var rep int
473+ var nb uint
474+ var b int
475+ switch x {
476+ default:
477+ return InternalError("unexpected length code")
478+ case 16:
479+ rep = 3
480+ nb = 2
481+ if i == 0 {
482+ return CorruptInputError(f.roffset)
483+ }
484+ b = f.bits[i-1]
485+ case 17:
486+ rep = 3
487+ nb = 3
488+ b = 0
489+ case 18:
490+ rep = 11
491+ nb = 7
492+ b = 0
493+ }
494+ for f.nb < nb {
495+ if err := f.moreBits(); err != nil {
496+ return err
497+ }
498+ }
499+ rep += int(f.b & uint32(1<<nb-1))
500+ f.b >>= nb
501+ f.nb -= nb
502+ if i+rep > n {
503+ return CorruptInputError(f.roffset)
504+ }
505+ for j := 0; j < rep; j++ {
506+ f.bits[i] = b
507+ i++
508+ }
509+ }
510+
511+ if !f.h1.init(f.bits[0:nlit]) || !f.h2.init(f.bits[nlit:nlit+ndist]) {
512+ return CorruptInputError(f.roffset)
513+ }
514+
515+ // As an optimization, we can initialize the min bits to read at a time
516+ // for the HLIT tree to the length of the EOB marker since we know that
517+ // every block must terminate with one. This preserves the property that
518+ // we never read any extra bytes after the end of the DEFLATE stream.
519+ if f.h1.min < f.bits[endBlockMarker] {
520+ f.h1.min = f.bits[endBlockMarker]
521+ }
522+
523+ return nil
524+}
525+
526+// Decode a single Huffman block from f.
527+// hl and hd are the Huffman states for the lit/length values
528+// and the distance values, respectively. If hd == nil, using the
529+// fixed distance encoding associated with fixed Huffman blocks.
530+func (f *decompressor) huffmanBlock() {
531+ const (
532+ stateInit = iota // Zero value must be stateInit
533+ stateDict
534+ )
535+
536+ switch f.stepState {
537+ case stateInit:
538+ goto readLiteral
539+ case stateDict:
540+ goto copyHistory
541+ }
542+
543+readLiteral:
544+ // Read literal and/or (length, distance) according to RFC section 3.2.3.
545+ {
546+ v, err := f.huffSym(f.hl)
547+ if err != nil {
548+ f.err = err
549+ return
550+ }
551+ var n uint // number of bits extra
552+ var length int
553+ switch {
554+ case v < 256:
555+ f.dict.writeByte(byte(v))
556+ if f.dict.availWrite() == 0 {
557+ f.toRead = f.dict.readFlush()
558+ f.step = (*decompressor).huffmanBlock
559+ f.stepState = stateInit
560+ return
561+ }
562+ goto readLiteral
563+ case v == 256:
564+ f.finishBlock()
565+ return
566+ // otherwise, reference to older data
567+ case v < 265:
568+ length = v - (257 - 3)
569+ n = 0
570+ case v < 269:
571+ length = v*2 - (265*2 - 11)
572+ n = 1
573+ case v < 273:
574+ length = v*4 - (269*4 - 19)
575+ n = 2
576+ case v < 277:
577+ length = v*8 - (273*8 - 35)
578+ n = 3
579+ case v < 281:
580+ length = v*16 - (277*16 - 67)
581+ n = 4
582+ case v < 285:
583+ length = v*32 - (281*32 - 131)
584+ n = 5
585+ case v < maxNumLit:
586+ length = 258
587+ n = 0
588+ default:
589+ f.err = CorruptInputError(f.roffset)
590+ return
591+ }
592+ if n > 0 {
593+ for f.nb < n {
594+ if err = f.moreBits(); err != nil {
595+ f.err = err
596+ return
597+ }
598+ }
599+ length += int(f.b & uint32(1<<n-1))
600+ f.b >>= n
601+ f.nb -= n
602+ }
603+
604+ var dist int
605+ if f.hd == nil {
606+ for f.nb < 5 {
607+ if err = f.moreBits(); err != nil {
608+ f.err = err
609+ return
610+ }
611+ }
612+ dist = int(reverseByte[(f.b&0x1F)<<3])
613+ f.b >>= 5
614+ f.nb -= 5
615+ } else {
616+ if dist, err = f.huffSym(f.hd); err != nil {
617+ f.err = err
618+ return
619+ }
620+ }
621+
622+ switch {
623+ case dist < 4:
624+ dist++
625+ case dist < maxNumDist:
626+ nb := uint(dist-2) >> 1
627+ // have 1 bit in bottom of dist, need nb more.
628+ extra := (dist & 1) << nb
629+ for f.nb < nb {
630+ if err = f.moreBits(); err != nil {
631+ f.err = err
632+ return
633+ }
634+ }
635+ extra |= int(f.b & uint32(1<<nb-1))
636+ f.b >>= nb
637+ f.nb -= nb
638+ dist = 1<<(nb+1) + 1 + extra
639+ default:
640+ f.err = CorruptInputError(f.roffset)
641+ return
642+ }
643+
644+ // No check on length; encoding can be prescient.
645+ if dist > f.dict.histSize() {
646+ f.err = CorruptInputError(f.roffset)
647+ return
648+ }
649+
650+ f.copyLen, f.copyDist = length, dist
651+ goto copyHistory
652+ }
653+
654+copyHistory:
655+ // Perform a backwards copy according to RFC section 3.2.3.
656+ {
657+ cnt := f.dict.tryWriteCopy(f.copyDist, f.copyLen)
658+ if cnt == 0 {
659+ cnt = f.dict.writeCopy(f.copyDist, f.copyLen)
660+ }
661+ f.copyLen -= cnt
662+
663+ if f.dict.availWrite() == 0 || f.copyLen > 0 {
664+ f.toRead = f.dict.readFlush()
665+ f.step = (*decompressor).huffmanBlock // We need to continue this work
666+ f.stepState = stateDict
667+ return
668+ }
669+ goto readLiteral
670+ }
671+}
672+
673+// Copy a single uncompressed data block from input to output.
674+func (f *decompressor) dataBlock() {
675+ // Uncompressed.
676+ // Discard current half-byte.
677+ f.nb = 0
678+ f.b = 0
679+
680+ // Length then ones-complement of length.
681+ nr, err := io.ReadFull(f.r, f.buf[0:4])
682+ f.roffset += int64(nr)
683+ if err != nil {
684+ if err == io.EOF {
685+ err = io.ErrUnexpectedEOF
686+ }
687+ f.err = err
688+ return
689+ }
690+ n := int(f.buf[0]) | int(f.buf[1])<<8
691+ nn := int(f.buf[2]) | int(f.buf[3])<<8
692+ if uint16(nn) != uint16(^n) {
693+ f.err = CorruptInputError(f.roffset)
694+ return
695+ }
696+
697+ if n == 0 {
698+ f.toRead = f.dict.readFlush()
699+ f.finishBlock()
700+ return
701+ }
702+
703+ f.copyLen = n
704+ f.copyData()
705+}
706+
707+// copyData copies f.copyLen bytes from the underlying reader into f.hist.
708+// It pauses for reads when f.hist is full.
709+func (f *decompressor) copyData() {
710+ buf := f.dict.writeSlice()
711+ if len(buf) > f.copyLen {
712+ buf = buf[:f.copyLen]
713+ }
714+
715+ cnt, err := io.ReadFull(f.r, buf)
716+ f.roffset += int64(cnt)
717+ f.copyLen -= cnt
718+ f.dict.writeMark(cnt)
719+ if err != nil {
720+ if err == io.EOF {
721+ err = io.ErrUnexpectedEOF
722+ }
723+ f.err = err
724+ return
725+ }
726+
727+ if f.dict.availWrite() == 0 || f.copyLen > 0 {
728+ f.toRead = f.dict.readFlush()
729+ f.step = (*decompressor).copyData
730+ return
731+ }
732+ f.finishBlock()
733+}
734+
735+func (f *decompressor) finishBlock() {
736+ if f.final {
737+ if f.dict.availRead() > 0 {
738+ f.toRead = f.dict.readFlush()
739+ }
740+ f.err = io.EOF
741+ }
742+ f.step = (*decompressor).nextBlock
743+}
744+
745+func (f *decompressor) moreBits() error {
746+ c, err := f.r.ReadByte()
747+ if err != nil {
748+ if err == io.EOF {
749+ err = io.ErrUnexpectedEOF
750+ }
751+ return err
752+ }
753+ f.roffset++
754+ f.b |= uint32(c) << f.nb
755+ f.nb += 8
756+ return nil
757+}
758+
759+// Read the next Huffman-encoded symbol from f according to h.
760+func (f *decompressor) huffSym(h *huffmanDecoder) (int, error) {
761+ // Since a huffmanDecoder can be empty or be composed of a degenerate tree
762+ // with single element, huffSym must error on these two edge cases. In both
763+ // cases, the chunks slice will be 0 for the invalid sequence, leading it
764+ // satisfy the n == 0 check below.
765+ n := uint(h.min)
766+ for {
767+ for f.nb < n {
768+ if err := f.moreBits(); err != nil {
769+ return 0, err
770+ }
771+ }
772+ chunk := h.chunks[f.b&(huffmanNumChunks-1)]
773+ n = uint(chunk & huffmanCountMask)
774+ if n > huffmanChunkBits {
775+ chunk = h.links[chunk>>huffmanValueShift][(f.b>>huffmanChunkBits)&h.linkMask]
776+ n = uint(chunk & huffmanCountMask)
777+ }
778+ if n <= f.nb {
779+ if n == 0 {
780+ f.err = CorruptInputError(f.roffset)
781+ return 0, f.err
782+ }
783+ f.b >>= n
784+ f.nb -= n
785+ return int(chunk >> huffmanValueShift), nil
786+ }
787+ }
788+}
789+
790+func makeReader(r io.Reader) Reader {
791+ if rr, ok := r.(Reader); ok {
792+ return rr
793+ }
794+ return bufio.NewReader(r)
795+}
796+
797+func fixedHuffmanDecoderInit() {
798+ fixedOnce.Do(func() {
799+ // These come from the RFC section 3.2.6.
800+ var bits [288]int
801+ for i := 0; i < 144; i++ {
802+ bits[i] = 8
803+ }
804+ for i := 144; i < 256; i++ {
805+ bits[i] = 9
806+ }
807+ for i := 256; i < 280; i++ {
808+ bits[i] = 7
809+ }
810+ for i := 280; i < 288; i++ {
811+ bits[i] = 8
812+ }
813+ fixedHuffmanDecoder.init(bits[:])
814+ })
815+}
816+
817+func (f *decompressor) Reset(r io.Reader, dict []byte) error {
818+ *f = decompressor{
819+ r: makeReader(r),
820+ bits: f.bits,
821+ codebits: f.codebits,
822+ h1: f.h1,
823+ h2: f.h2,
824+ dict: f.dict,
825+ step: (*decompressor).nextBlock,
826+ }
827+ f.dict.init(maxMatchOffset, dict)
828+ return nil
829+}
830+
831+// NewReader returns a new ReadCloser that can be used
832+// to read the uncompressed version of r.
833+// If r does not also implement io.ByteReader,
834+// the decompressor may read more data than necessary from r.
835+// It is the caller's responsibility to call Close on the ReadCloser
836+// when finished reading.
837+//
838+// The ReadCloser returned by NewReader also implements Resetter.
839+func NewReader(r io.Reader) io.ReadCloser {
840+ fixedHuffmanDecoderInit()
841+
842+ var f decompressor
843+ f.r = makeReader(r)
844+ f.bits = new([maxNumLit + maxNumDist]int)
845+ f.codebits = new([numCodes]int)
846+ f.step = (*decompressor).nextBlock
847+ f.dict.init(maxMatchOffset, nil)
848+ return &f
849+}
850+
851+// NewReaderDict is like NewReader but initializes the reader
852+// with a preset dictionary. The returned Reader behaves as if
853+// the uncompressed data stream started with the given dictionary,
854+// which has already been read. NewReaderDict is typically used
855+// to read data compressed by NewWriterDict.
856+//
857+// The ReadCloser returned by NewReader also implements Resetter.
858+func NewReaderDict(r io.Reader, dict []byte) io.ReadCloser {
859+ fixedHuffmanDecoderInit()
860+
861+ var f decompressor
862+ f.r = makeReader(r)
863+ f.bits = new([maxNumLit + maxNumDist]int)
864+ f.codebits = new([numCodes]int)
865+ f.step = (*decompressor).nextBlock
866+ f.dict.init(maxMatchOffset, dict)
867+ return &f
868+}
vendor/github.com/klauspost/compress/flate/reverse_bits.go+48−0
@@ -0,0 +1,48 @@
1+// Copyright 2009 The Go Authors. All rights reserved.
2+// Use of this source code is governed by a BSD-style
3+// license that can be found in the LICENSE file.
4+
5+package flate
6+
7+var reverseByte = [256]byte{
8+ 0x00, 0x80, 0x40, 0xc0, 0x20, 0xa0, 0x60, 0xe0,
9+ 0x10, 0x90, 0x50, 0xd0, 0x30, 0xb0, 0x70, 0xf0,
10+ 0x08, 0x88, 0x48, 0xc8, 0x28, 0xa8, 0x68, 0xe8,
11+ 0x18, 0x98, 0x58, 0xd8, 0x38, 0xb8, 0x78, 0xf8,
12+ 0x04, 0x84, 0x44, 0xc4, 0x24, 0xa4, 0x64, 0xe4,
13+ 0x14, 0x94, 0x54, 0xd4, 0x34, 0xb4, 0x74, 0xf4,
14+ 0x0c, 0x8c, 0x4c, 0xcc, 0x2c, 0xac, 0x6c, 0xec,
15+ 0x1c, 0x9c, 0x5c, 0xdc, 0x3c, 0xbc, 0x7c, 0xfc,
16+ 0x02, 0x82, 0x42, 0xc2, 0x22, 0xa2, 0x62, 0xe2,
17+ 0x12, 0x92, 0x52, 0xd2, 0x32, 0xb2, 0x72, 0xf2,
18+ 0x0a, 0x8a, 0x4a, 0xca, 0x2a, 0xaa, 0x6a, 0xea,
19+ 0x1a, 0x9a, 0x5a, 0xda, 0x3a, 0xba, 0x7a, 0xfa,
20+ 0x06, 0x86, 0x46, 0xc6, 0x26, 0xa6, 0x66, 0xe6,
21+ 0x16, 0x96, 0x56, 0xd6, 0x36, 0xb6, 0x76, 0xf6,
22+ 0x0e, 0x8e, 0x4e, 0xce, 0x2e, 0xae, 0x6e, 0xee,
23+ 0x1e, 0x9e, 0x5e, 0xde, 0x3e, 0xbe, 0x7e, 0xfe,
24+ 0x01, 0x81, 0x41, 0xc1, 0x21, 0xa1, 0x61, 0xe1,
25+ 0x11, 0x91, 0x51, 0xd1, 0x31, 0xb1, 0x71, 0xf1,
26+ 0x09, 0x89, 0x49, 0xc9, 0x29, 0xa9, 0x69, 0xe9,
27+ 0x19, 0x99, 0x59, 0xd9, 0x39, 0xb9, 0x79, 0xf9,
28+ 0x05, 0x85, 0x45, 0xc5, 0x25, 0xa5, 0x65, 0xe5,
29+ 0x15, 0x95, 0x55, 0xd5, 0x35, 0xb5, 0x75, 0xf5,
30+ 0x0d, 0x8d, 0x4d, 0xcd, 0x2d, 0xad, 0x6d, 0xed,
31+ 0x1d, 0x9d, 0x5d, 0xdd, 0x3d, 0xbd, 0x7d, 0xfd,
32+ 0x03, 0x83, 0x43, 0xc3, 0x23, 0xa3, 0x63, 0xe3,
33+ 0x13, 0x93, 0x53, 0xd3, 0x33, 0xb3, 0x73, 0xf3,
34+ 0x0b, 0x8b, 0x4b, 0xcb, 0x2b, 0xab, 0x6b, 0xeb,
35+ 0x1b, 0x9b, 0x5b, 0xdb, 0x3b, 0xbb, 0x7b, 0xfb,
36+ 0x07, 0x87, 0x47, 0xc7, 0x27, 0xa7, 0x67, 0xe7,
37+ 0x17, 0x97, 0x57, 0xd7, 0x37, 0xb7, 0x77, 0xf7,
38+ 0x0f, 0x8f, 0x4f, 0xcf, 0x2f, 0xaf, 0x6f, 0xef,
39+ 0x1f, 0x9f, 0x5f, 0xdf, 0x3f, 0xbf, 0x7f, 0xff,
40+}
41+
42+func reverseUint16(v uint16) uint16 {
43+ return uint16(reverseByte[v>>8]) | uint16(reverseByte[v&0xFF])<<8
44+}
45+
46+func reverseBits(number uint16, bitLength byte) uint16 {
47+ return reverseUint16(number << uint8(16-bitLength))
48+}
vendor/github.com/klauspost/compress/flate/snappy.go+900−0
@@ -0,0 +1,900 @@
1+// Copyright 2011 The Snappy-Go Authors. All rights reserved.
2+// Modified for deflate by Klaus Post (c) 2015.
3+// Use of this source code is governed by a BSD-style
4+// license that can be found in the LICENSE file.
5+
6+package flate
7+
8+// emitLiteral writes a literal chunk and returns the number of bytes written.
9+func emitLiteral(dst *tokens, lit []byte) {
10+ ol := int(dst.n)
11+ for i, v := range lit {
12+ dst.tokens[(i+ol)&maxStoreBlockSize] = token(v)
13+ }
14+ dst.n += uint16(len(lit))
15+}
16+
17+// emitCopy writes a copy chunk and returns the number of bytes written.
18+func emitCopy(dst *tokens, offset, length int) {
19+ dst.tokens[dst.n] = matchToken(uint32(length-3), uint32(offset-minOffsetSize))
20+ dst.n++
21+}
22+
23+type snappyEnc interface {
24+ Encode(dst *tokens, src []byte)
25+ Reset()
26+}
27+
28+func newSnappy(level int) snappyEnc {
29+ switch level {
30+ case 1:
31+ return &snappyL1{}
32+ case 2:
33+ return &snappyL2{snappyGen: snappyGen{cur: maxStoreBlockSize, prev: make([]byte, 0, maxStoreBlockSize)}}
34+ case 3:
35+ return &snappyL3{snappyGen: snappyGen{cur: maxStoreBlockSize, prev: make([]byte, 0, maxStoreBlockSize)}}
36+ case 4:
37+ return &snappyL4{snappyL3{snappyGen: snappyGen{cur: maxStoreBlockSize, prev: make([]byte, 0, maxStoreBlockSize)}}}
38+ default:
39+ panic("invalid level specified")
40+ }
41+}
42+
43+const (
44+ tableBits = 14 // Bits used in the table
45+ tableSize = 1 << tableBits // Size of the table
46+ tableMask = tableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks.
47+ tableShift = 32 - tableBits // Right-shift to get the tableBits most significant bits of a uint32.
48+ baseMatchOffset = 1 // The smallest match offset
49+ baseMatchLength = 3 // The smallest match length per the RFC section 3.2.5
50+ maxMatchOffset = 1 << 15 // The largest match offset
51+)
52+
53+func load32(b []byte, i int) uint32 {
54+ b = b[i : i+4 : len(b)] // Help the compiler eliminate bounds checks on the next line.
55+ return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24
56+}
57+
58+func load64(b []byte, i int) uint64 {
59+ b = b[i : i+8 : len(b)] // Help the compiler eliminate bounds checks on the next line.
60+ return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 |
61+ uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56
62+}
63+
64+func hash(u uint32) uint32 {
65+ return (u * 0x1e35a7bd) >> tableShift
66+}
67+
68+// snappyL1 encapsulates level 1 compression
69+type snappyL1 struct{}
70+
71+func (e *snappyL1) Reset() {}
72+
73+func (e *snappyL1) Encode(dst *tokens, src []byte) {
74+ const (
75+ inputMargin = 16 - 1
76+ minNonLiteralBlockSize = 1 + 1 + inputMargin
77+ )
78+
79+ // This check isn't in the Snappy implementation, but there, the caller
80+ // instead of the callee handles this case.
81+ if len(src) < minNonLiteralBlockSize {
82+ // We do not fill the token table.
83+ // This will be picked up by caller.
84+ dst.n = uint16(len(src))
85+ return
86+ }
87+
88+ // Initialize the hash table.
89+ //
90+ // The table element type is uint16, as s < sLimit and sLimit < len(src)
91+ // and len(src) <= maxStoreBlockSize and maxStoreBlockSize == 65535.
92+ var table [tableSize]uint16
93+
94+ // sLimit is when to stop looking for offset/length copies. The inputMargin
95+ // lets us use a fast path for emitLiteral in the main loop, while we are
96+ // looking for copies.
97+ sLimit := len(src) - inputMargin
98+
99+ // nextEmit is where in src the next emitLiteral should start from.
100+ nextEmit := 0
101+
102+ // The encoded form must start with a literal, as there are no previous
103+ // bytes to copy, so we start looking for hash matches at s == 1.
104+ s := 1
105+ nextHash := hash(load32(src, s))
106+
107+ for {
108+ // Copied from the C++ snappy implementation:
109+ //
110+ // Heuristic match skipping: If 32 bytes are scanned with no matches
111+ // found, start looking only at every other byte. If 32 more bytes are
112+ // scanned (or skipped), look at every third byte, etc.. When a match
113+ // is found, immediately go back to looking at every byte. This is a
114+ // small loss (~5% performance, ~0.1% density) for compressible data
115+ // due to more bookkeeping, but for non-compressible data (such as
116+ // JPEG) it's a huge win since the compressor quickly "realizes" the
117+ // data is incompressible and doesn't bother looking for matches
118+ // everywhere.
119+ //
120+ // The "skip" variable keeps track of how many bytes there are since
121+ // the last match; dividing it by 32 (ie. right-shifting by five) gives
122+ // the number of bytes to move ahead for each iteration.
123+ skip := 32
124+
125+ nextS := s
126+ candidate := 0
127+ for {
128+ s = nextS
129+ bytesBetweenHashLookups := skip >> 5
130+ nextS = s + bytesBetweenHashLookups
131+ skip += bytesBetweenHashLookups
132+ if nextS > sLimit {
133+ goto emitRemainder
134+ }
135+ candidate = int(table[nextHash&tableMask])
136+ table[nextHash&tableMask] = uint16(s)
137+ nextHash = hash(load32(src, nextS))
138+ if s-candidate <= maxMatchOffset && load32(src, s) == load32(src, candidate) {
139+ break
140+ }
141+ }
142+
143+ // A 4-byte match has been found. We'll later see if more than 4 bytes
144+ // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
145+ // them as literal bytes.
146+ emitLiteral(dst, src[nextEmit:s])
147+
148+ // Call emitCopy, and then see if another emitCopy could be our next
149+ // move. Repeat until we find no match for the input immediately after
150+ // what was consumed by the last emitCopy call.
151+ //
152+ // If we exit this loop normally then we need to call emitLiteral next,
153+ // though we don't yet know how big the literal will be. We handle that
154+ // by proceeding to the next iteration of the main loop. We also can
155+ // exit this loop via goto if we get close to exhausting the input.
156+ for {
157+ // Invariant: we have a 4-byte match at s, and no need to emit any
158+ // literal bytes prior to s.
159+ base := s
160+
161+ // Extend the 4-byte match as long as possible.
162+ //
163+ // This is an inlined version of Snappy's:
164+ // s = extendMatch(src, candidate+4, s+4)
165+ s += 4
166+ s1 := base + maxMatchLength
167+ if s1 > len(src) {
168+ s1 = len(src)
169+ }
170+ a := src[s:s1]
171+ b := src[candidate+4:]
172+ b = b[:len(a)]
173+ l := len(a)
174+ for i := range a {
175+ if a[i] != b[i] {
176+ l = i
177+ break
178+ }
179+ }
180+ s += l
181+
182+ // matchToken is flate's equivalent of Snappy's emitCopy.
183+ dst.tokens[dst.n] = matchToken(uint32(s-base-baseMatchLength), uint32(base-candidate-baseMatchOffset))
184+ dst.n++
185+ nextEmit = s
186+ if s >= sLimit {
187+ goto emitRemainder
188+ }
189+
190+ // We could immediately start working at s now, but to improve
191+ // compression we first update the hash table at s-1 and at s. If
192+ // another emitCopy is not our next move, also calculate nextHash
193+ // at s+1. At least on GOARCH=amd64, these three hash calculations
194+ // are faster as one load64 call (with some shifts) instead of
195+ // three load32 calls.
196+ x := load64(src, s-1)
197+ prevHash := hash(uint32(x >> 0))
198+ table[prevHash&tableMask] = uint16(s - 1)
199+ currHash := hash(uint32(x >> 8))
200+ candidate = int(table[currHash&tableMask])
201+ table[currHash&tableMask] = uint16(s)
202+ if s-candidate > maxMatchOffset || uint32(x>>8) != load32(src, candidate) {
203+ nextHash = hash(uint32(x >> 16))
204+ s++
205+ break
206+ }
207+ }
208+ }
209+
210+emitRemainder:
211+ if nextEmit < len(src) {
212+ emitLiteral(dst, src[nextEmit:])
213+ }
214+}
215+
216+type tableEntry struct {
217+ val uint32
218+ offset int32
219+}
220+
221+func load3232(b []byte, i int32) uint32 {
222+ b = b[i : i+4 : len(b)] // Help the compiler eliminate bounds checks on the next line.
223+ return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24
224+}
225+
226+func load6432(b []byte, i int32) uint64 {
227+ b = b[i : i+8 : len(b)] // Help the compiler eliminate bounds checks on the next line.
228+ return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 |
229+ uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56
230+}
231+
232+// snappyGen maintains the table for matches,
233+// and the previous byte block for level 2.
234+// This is the generic implementation.
235+type snappyGen struct {
236+ prev []byte
237+ cur int32
238+}
239+
240+// snappyGen maintains the table for matches,
241+// and the previous byte block for level 2.
242+// This is the generic implementation.
243+type snappyL2 struct {
244+ snappyGen
245+ table [tableSize]tableEntry
246+}
247+
248+// EncodeL2 uses a similar algorithm to level 1, but is capable
249+// of matching across blocks giving better compression at a small slowdown.
250+func (e *snappyL2) Encode(dst *tokens, src []byte) {
251+ const (
252+ inputMargin = 8 - 1
253+ minNonLiteralBlockSize = 1 + 1 + inputMargin
254+ )
255+
256+ // Protect against e.cur wraparound.
257+ if e.cur > 1<<30 {
258+ for i := range e.table[:] {
259+ e.table[i] = tableEntry{}
260+ }
261+ e.cur = maxStoreBlockSize
262+ }
263+
264+ // This check isn't in the Snappy implementation, but there, the caller
265+ // instead of the callee handles this case.
266+ if len(src) < minNonLiteralBlockSize {
267+ // We do not fill the token table.
268+ // This will be picked up by caller.
269+ dst.n = uint16(len(src))
270+ e.cur += maxStoreBlockSize
271+ e.prev = e.prev[:0]
272+ return
273+ }
274+
275+ // sLimit is when to stop looking for offset/length copies. The inputMargin
276+ // lets us use a fast path for emitLiteral in the main loop, while we are
277+ // looking for copies.
278+ sLimit := int32(len(src) - inputMargin)
279+
280+ // nextEmit is where in src the next emitLiteral should start from.
281+ nextEmit := int32(0)
282+ s := int32(0)
283+ cv := load3232(src, s)
284+ nextHash := hash(cv)
285+
286+ for {
287+ // Copied from the C++ snappy implementation:
288+ //
289+ // Heuristic match skipping: If 32 bytes are scanned with no matches
290+ // found, start looking only at every other byte. If 32 more bytes are
291+ // scanned (or skipped), look at every third byte, etc.. When a match
292+ // is found, immediately go back to looking at every byte. This is a
293+ // small loss (~5% performance, ~0.1% density) for compressible data
294+ // due to more bookkeeping, but for non-compressible data (such as
295+ // JPEG) it's a huge win since the compressor quickly "realizes" the
296+ // data is incompressible and doesn't bother looking for matches
297+ // everywhere.
298+ //
299+ // The "skip" variable keeps track of how many bytes there are since
300+ // the last match; dividing it by 32 (ie. right-shifting by five) gives
301+ // the number of bytes to move ahead for each iteration.
302+ skip := int32(32)
303+
304+ nextS := s
305+ var candidate tableEntry
306+ for {
307+ s = nextS
308+ bytesBetweenHashLookups := skip >> 5
309+ nextS = s + bytesBetweenHashLookups
310+ skip += bytesBetweenHashLookups
311+ if nextS > sLimit {
312+ goto emitRemainder
313+ }
314+ candidate = e.table[nextHash&tableMask]
315+ now := load3232(src, nextS)
316+ e.table[nextHash&tableMask] = tableEntry{offset: s + e.cur, val: cv}
317+ nextHash = hash(now)
318+
319+ offset := s - (candidate.offset - e.cur)
320+ if offset > maxMatchOffset || cv != candidate.val {
321+ // Out of range or not matched.
322+ cv = now
323+ continue
324+ }
325+ break
326+ }
327+
328+ // A 4-byte match has been found. We'll later see if more than 4 bytes
329+ // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
330+ // them as literal bytes.
331+ emitLiteral(dst, src[nextEmit:s])
332+
333+ // Call emitCopy, and then see if another emitCopy could be our next
334+ // move. Repeat until we find no match for the input immediately after
335+ // what was consumed by the last emitCopy call.
336+ //
337+ // If we exit this loop normally then we need to call emitLiteral next,
338+ // though we don't yet know how big the literal will be. We handle that
339+ // by proceeding to the next iteration of the main loop. We also can
340+ // exit this loop via goto if we get close to exhausting the input.
341+ for {
342+ // Invariant: we have a 4-byte match at s, and no need to emit any
343+ // literal bytes prior to s.
344+
345+ // Extend the 4-byte match as long as possible.
346+ //
347+ s += 4
348+ t := candidate.offset - e.cur + 4
349+ l := e.matchlen(s, t, src)
350+
351+ // matchToken is flate's equivalent of Snappy's emitCopy. (length,offset)
352+ dst.tokens[dst.n] = matchToken(uint32(l+4-baseMatchLength), uint32(s-t-baseMatchOffset))
353+ dst.n++
354+ s += l
355+ nextEmit = s
356+ if s >= sLimit {
357+ t += l
358+ // Index first pair after match end.
359+ if int(t+4) < len(src) && t > 0 {
360+ cv := load3232(src, t)
361+ e.table[hash(cv)&tableMask] = tableEntry{offset: t + e.cur, val: cv}
362+ }
363+ goto emitRemainder
364+ }
365+
366+ // We could immediately start working at s now, but to improve
367+ // compression we first update the hash table at s-1 and at s. If
368+ // another emitCopy is not our next move, also calculate nextHash
369+ // at s+1. At least on GOARCH=amd64, these three hash calculations
370+ // are faster as one load64 call (with some shifts) instead of
371+ // three load32 calls.
372+ x := load6432(src, s-1)
373+ prevHash := hash(uint32(x))
374+ e.table[prevHash&tableMask] = tableEntry{offset: e.cur + s - 1, val: uint32(x)}
375+ x >>= 8
376+ currHash := hash(uint32(x))
377+ candidate = e.table[currHash&tableMask]
378+ e.table[currHash&tableMask] = tableEntry{offset: e.cur + s, val: uint32(x)}
379+
380+ offset := s - (candidate.offset - e.cur)
381+ if offset > maxMatchOffset || uint32(x) != candidate.val {
382+ cv = uint32(x >> 8)
383+ nextHash = hash(cv)
384+ s++
385+ break
386+ }
387+ }
388+ }
389+
390+emitRemainder:
391+ if int(nextEmit) < len(src) {
392+ emitLiteral(dst, src[nextEmit:])
393+ }
394+ e.cur += int32(len(src))
395+ e.prev = e.prev[:len(src)]
396+ copy(e.prev, src)
397+}
398+
399+type tableEntryPrev struct {
400+ Cur tableEntry
401+ Prev tableEntry
402+}
403+
404+// snappyL3
405+type snappyL3 struct {
406+ snappyGen
407+ table [tableSize]tableEntryPrev
408+}
409+
410+// Encode uses a similar algorithm to level 2, will check up to two candidates.
411+func (e *snappyL3) Encode(dst *tokens, src []byte) {
412+ const (
413+ inputMargin = 8 - 1
414+ minNonLiteralBlockSize = 1 + 1 + inputMargin
415+ )
416+
417+ // Protect against e.cur wraparound.
418+ if e.cur > 1<<30 {
419+ for i := range e.table[:] {
420+ e.table[i] = tableEntryPrev{}
421+ }
422+ e.snappyGen = snappyGen{cur: maxStoreBlockSize, prev: e.prev[:0]}
423+ }
424+
425+ // This check isn't in the Snappy implementation, but there, the caller
426+ // instead of the callee handles this case.
427+ if len(src) < minNonLiteralBlockSize {
428+ // We do not fill the token table.
429+ // This will be picked up by caller.
430+ dst.n = uint16(len(src))
431+ e.cur += maxStoreBlockSize
432+ e.prev = e.prev[:0]
433+ return
434+ }
435+
436+ // sLimit is when to stop looking for offset/length copies. The inputMargin
437+ // lets us use a fast path for emitLiteral in the main loop, while we are
438+ // looking for copies.
439+ sLimit := int32(len(src) - inputMargin)
440+
441+ // nextEmit is where in src the next emitLiteral should start from.
442+ nextEmit := int32(0)
443+ s := int32(0)
444+ cv := load3232(src, s)
445+ nextHash := hash(cv)
446+
447+ for {
448+ // Copied from the C++ snappy implementation:
449+ //
450+ // Heuristic match skipping: If 32 bytes are scanned with no matches
451+ // found, start looking only at every other byte. If 32 more bytes are
452+ // scanned (or skipped), look at every third byte, etc.. When a match
453+ // is found, immediately go back to looking at every byte. This is a
454+ // small loss (~5% performance, ~0.1% density) for compressible data
455+ // due to more bookkeeping, but for non-compressible data (such as
456+ // JPEG) it's a huge win since the compressor quickly "realizes" the
457+ // data is incompressible and doesn't bother looking for matches
458+ // everywhere.
459+ //
460+ // The "skip" variable keeps track of how many bytes there are since
461+ // the last match; dividing it by 32 (ie. right-shifting by five) gives
462+ // the number of bytes to move ahead for each iteration.
463+ skip := int32(32)
464+
465+ nextS := s
466+ var candidate tableEntry
467+ for {
468+ s = nextS
469+ bytesBetweenHashLookups := skip >> 5
470+ nextS = s + bytesBetweenHashLookups
471+ skip += bytesBetweenHashLookups
472+ if nextS > sLimit {
473+ goto emitRemainder
474+ }
475+ candidates := e.table[nextHash&tableMask]
476+ now := load3232(src, nextS)
477+ e.table[nextHash&tableMask] = tableEntryPrev{Prev: candidates.Cur, Cur: tableEntry{offset: s + e.cur, val: cv}}
478+ nextHash = hash(now)
479+
480+ // Check both candidates
481+ candidate = candidates.Cur
482+ if cv == candidate.val {
483+ offset := s - (candidate.offset - e.cur)
484+ if offset <= maxMatchOffset {
485+ break
486+ }
487+ } else {
488+ // We only check if value mismatches.
489+ // Offset will always be invalid in other cases.
490+ candidate = candidates.Prev
491+ if cv == candidate.val {
492+ offset := s - (candidate.offset - e.cur)
493+ if offset <= maxMatchOffset {
494+ break
495+ }
496+ }
497+ }
498+ cv = now
499+ }
500+
501+ // A 4-byte match has been found. We'll later see if more than 4 bytes
502+ // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
503+ // them as literal bytes.
504+ emitLiteral(dst, src[nextEmit:s])
505+
506+ // Call emitCopy, and then see if another emitCopy could be our next
507+ // move. Repeat until we find no match for the input immediately after
508+ // what was consumed by the last emitCopy call.
509+ //
510+ // If we exit this loop normally then we need to call emitLiteral next,
511+ // though we don't yet know how big the literal will be. We handle that
512+ // by proceeding to the next iteration of the main loop. We also can
513+ // exit this loop via goto if we get close to exhausting the input.
514+ for {
515+ // Invariant: we have a 4-byte match at s, and no need to emit any
516+ // literal bytes prior to s.
517+
518+ // Extend the 4-byte match as long as possible.
519+ //
520+ s += 4
521+ t := candidate.offset - e.cur + 4
522+ l := e.matchlen(s, t, src)
523+
524+ // matchToken is flate's equivalent of Snappy's emitCopy. (length,offset)
525+ dst.tokens[dst.n] = matchToken(uint32(l+4-baseMatchLength), uint32(s-t-baseMatchOffset))
526+ dst.n++
527+ s += l
528+ nextEmit = s
529+ if s >= sLimit {
530+ t += l
531+ // Index first pair after match end.
532+ if int(t+4) < len(src) && t > 0 {
533+ cv := load3232(src, t)
534+ nextHash = hash(cv)
535+ e.table[nextHash&tableMask] = tableEntryPrev{
536+ Prev: e.table[nextHash&tableMask].Cur,
537+ Cur: tableEntry{offset: e.cur + t, val: cv},
538+ }
539+ }
540+ goto emitRemainder
541+ }
542+
543+ // We could immediately start working at s now, but to improve
544+ // compression we first update the hash table at s-3 to s. If
545+ // another emitCopy is not our next move, also calculate nextHash
546+ // at s+1. At least on GOARCH=amd64, these three hash calculations
547+ // are faster as one load64 call (with some shifts) instead of
548+ // three load32 calls.
549+ x := load6432(src, s-3)
550+ prevHash := hash(uint32(x))
551+ e.table[prevHash&tableMask] = tableEntryPrev{
552+ Prev: e.table[prevHash&tableMask].Cur,
553+ Cur: tableEntry{offset: e.cur + s - 3, val: uint32(x)},
554+ }
555+ x >>= 8
556+ prevHash = hash(uint32(x))
557+
558+ e.table[prevHash&tableMask] = tableEntryPrev{
559+ Prev: e.table[prevHash&tableMask].Cur,
560+ Cur: tableEntry{offset: e.cur + s - 2, val: uint32(x)},
561+ }
562+ x >>= 8
563+ prevHash = hash(uint32(x))
564+
565+ e.table[prevHash&tableMask] = tableEntryPrev{
566+ Prev: e.table[prevHash&tableMask].Cur,
567+ Cur: tableEntry{offset: e.cur + s - 1, val: uint32(x)},
568+ }
569+ x >>= 8
570+ currHash := hash(uint32(x))
571+ candidates := e.table[currHash&tableMask]
572+ cv = uint32(x)
573+ e.table[currHash&tableMask] = tableEntryPrev{
574+ Prev: candidates.Cur,
575+ Cur: tableEntry{offset: s + e.cur, val: cv},
576+ }
577+
578+ // Check both candidates
579+ candidate = candidates.Cur
580+ if cv == candidate.val {
581+ offset := s - (candidate.offset - e.cur)
582+ if offset <= maxMatchOffset {
583+ continue
584+ }
585+ } else {
586+ // We only check if value mismatches.
587+ // Offset will always be invalid in other cases.
588+ candidate = candidates.Prev
589+ if cv == candidate.val {
590+ offset := s - (candidate.offset - e.cur)
591+ if offset <= maxMatchOffset {
592+ continue
593+ }
594+ }
595+ }
596+ cv = uint32(x >> 8)
597+ nextHash = hash(cv)
598+ s++
599+ break
600+ }
601+ }
602+
603+emitRemainder:
604+ if int(nextEmit) < len(src) {
605+ emitLiteral(dst, src[nextEmit:])
606+ }
607+ e.cur += int32(len(src))
608+ e.prev = e.prev[:len(src)]
609+ copy(e.prev, src)
610+}
611+
612+// snappyL4
613+type snappyL4 struct {
614+ snappyL3
615+}
616+
617+// Encode uses a similar algorithm to level 3,
618+// but will check up to two candidates if first isn't long enough.
619+func (e *snappyL4) Encode(dst *tokens, src []byte) {
620+ const (
621+ inputMargin = 8 - 3
622+ minNonLiteralBlockSize = 1 + 1 + inputMargin
623+ matchLenGood = 12
624+ )
625+
626+ // Protect against e.cur wraparound.
627+ if e.cur > 1<<30 {
628+ for i := range e.table[:] {
629+ e.table[i] = tableEntryPrev{}
630+ }
631+ e.snappyGen = snappyGen{cur: maxStoreBlockSize, prev: e.prev[:0]}
632+ }
633+
634+ // This check isn't in the Snappy implementation, but there, the caller
635+ // instead of the callee handles this case.
636+ if len(src) < minNonLiteralBlockSize {
637+ // We do not fill the token table.
638+ // This will be picked up by caller.
639+ dst.n = uint16(len(src))
640+ e.cur += maxStoreBlockSize
641+ e.prev = e.prev[:0]
642+ return
643+ }
644+
645+ // sLimit is when to stop looking for offset/length copies. The inputMargin
646+ // lets us use a fast path for emitLiteral in the main loop, while we are
647+ // looking for copies.
648+ sLimit := int32(len(src) - inputMargin)
649+
650+ // nextEmit is where in src the next emitLiteral should start from.
651+ nextEmit := int32(0)
652+ s := int32(0)
653+ cv := load3232(src, s)
654+ nextHash := hash(cv)
655+
656+ for {
657+ // Copied from the C++ snappy implementation:
658+ //
659+ // Heuristic match skipping: If 32 bytes are scanned with no matches
660+ // found, start looking only at every other byte. If 32 more bytes are
661+ // scanned (or skipped), look at every third byte, etc.. When a match
662+ // is found, immediately go back to looking at every byte. This is a
663+ // small loss (~5% performance, ~0.1% density) for compressible data
664+ // due to more bookkeeping, but for non-compressible data (such as
665+ // JPEG) it's a huge win since the compressor quickly "realizes" the
666+ // data is incompressible and doesn't bother looking for matches
667+ // everywhere.
668+ //
669+ // The "skip" variable keeps track of how many bytes there are since
670+ // the last match; dividing it by 32 (ie. right-shifting by five) gives
671+ // the number of bytes to move ahead for each iteration.
672+ skip := int32(32)
673+
674+ nextS := s
675+ var candidate tableEntry
676+ var candidateAlt tableEntry
677+ for {
678+ s = nextS
679+ bytesBetweenHashLookups := skip >> 5
680+ nextS = s + bytesBetweenHashLookups
681+ skip += bytesBetweenHashLookups
682+ if nextS > sLimit {
683+ goto emitRemainder
684+ }
685+ candidates := e.table[nextHash&tableMask]
686+ now := load3232(src, nextS)
687+ e.table[nextHash&tableMask] = tableEntryPrev{Prev: candidates.Cur, Cur: tableEntry{offset: s + e.cur, val: cv}}
688+ nextHash = hash(now)
689+
690+ // Check both candidates
691+ candidate = candidates.Cur
692+ if cv == candidate.val {
693+ offset := s - (candidate.offset - e.cur)
694+ if offset < maxMatchOffset {
695+ offset = s - (candidates.Prev.offset - e.cur)
696+ if cv == candidates.Prev.val && offset < maxMatchOffset {
697+ candidateAlt = candidates.Prev
698+ }
699+ break
700+ }
701+ } else {
702+ // We only check if value mismatches.
703+ // Offset will always be invalid in other cases.
704+ candidate = candidates.Prev
705+ if cv == candidate.val {
706+ offset := s - (candidate.offset - e.cur)
707+ if offset < maxMatchOffset {
708+ break
709+ }
710+ }
711+ }
712+ cv = now
713+ }
714+
715+ // A 4-byte match has been found. We'll later see if more than 4 bytes
716+ // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
717+ // them as literal bytes.
718+ emitLiteral(dst, src[nextEmit:s])
719+
720+ // Call emitCopy, and then see if another emitCopy could be our next
721+ // move. Repeat until we find no match for the input immediately after
722+ // what was consumed by the last emitCopy call.
723+ //
724+ // If we exit this loop normally then we need to call emitLiteral next,
725+ // though we don't yet know how big the literal will be. We handle that
726+ // by proceeding to the next iteration of the main loop. We also can
727+ // exit this loop via goto if we get close to exhausting the input.
728+ for {
729+ // Invariant: we have a 4-byte match at s, and no need to emit any
730+ // literal bytes prior to s.
731+
732+ // Extend the 4-byte match as long as possible.
733+ //
734+ s += 4
735+ t := candidate.offset - e.cur + 4
736+ l := e.matchlen(s, t, src)
737+ // Try alternative candidate if match length < matchLenGood.
738+ if l < matchLenGood-4 && candidateAlt.offset != 0 {
739+ t2 := candidateAlt.offset - e.cur + 4
740+ l2 := e.matchlen(s, t2, src)
741+ if l2 > l {
742+ l = l2
743+ t = t2
744+ }
745+ }
746+ // matchToken is flate's equivalent of Snappy's emitCopy. (length,offset)
747+ dst.tokens[dst.n] = matchToken(uint32(l+4-baseMatchLength), uint32(s-t-baseMatchOffset))
748+ dst.n++
749+ s += l
750+ nextEmit = s
751+ if s >= sLimit {
752+ t += l
753+ // Index first pair after match end.
754+ if int(t+4) < len(src) && t > 0 {
755+ cv := load3232(src, t)
756+ nextHash = hash(cv)
757+ e.table[nextHash&tableMask] = tableEntryPrev{
758+ Prev: e.table[nextHash&tableMask].Cur,
759+ Cur: tableEntry{offset: e.cur + t, val: cv},
760+ }
761+ }
762+ goto emitRemainder
763+ }
764+
765+ // We could immediately start working at s now, but to improve
766+ // compression we first update the hash table at s-3 to s. If
767+ // another emitCopy is not our next move, also calculate nextHash
768+ // at s+1. At least on GOARCH=amd64, these three hash calculations
769+ // are faster as one load64 call (with some shifts) instead of
770+ // three load32 calls.
771+ x := load6432(src, s-3)
772+ prevHash := hash(uint32(x))
773+ e.table[prevHash&tableMask] = tableEntryPrev{
774+ Prev: e.table[prevHash&tableMask].Cur,
775+ Cur: tableEntry{offset: e.cur + s - 3, val: uint32(x)},
776+ }
777+ x >>= 8
778+ prevHash = hash(uint32(x))
779+
780+ e.table[prevHash&tableMask] = tableEntryPrev{
781+ Prev: e.table[prevHash&tableMask].Cur,
782+ Cur: tableEntry{offset: e.cur + s - 2, val: uint32(x)},
783+ }
784+ x >>= 8
785+ prevHash = hash(uint32(x))
786+
787+ e.table[prevHash&tableMask] = tableEntryPrev{
788+ Prev: e.table[prevHash&tableMask].Cur,
789+ Cur: tableEntry{offset: e.cur + s - 1, val: uint32(x)},
790+ }
791+ x >>= 8
792+ currHash := hash(uint32(x))
793+ candidates := e.table[currHash&tableMask]
794+ cv = uint32(x)
795+ e.table[currHash&tableMask] = tableEntryPrev{
796+ Prev: candidates.Cur,
797+ Cur: tableEntry{offset: s + e.cur, val: cv},
798+ }
799+
800+ // Check both candidates
801+ candidate = candidates.Cur
802+ candidateAlt = tableEntry{}
803+ if cv == candidate.val {
804+ offset := s - (candidate.offset - e.cur)
805+ if offset <= maxMatchOffset {
806+ offset = s - (candidates.Prev.offset - e.cur)
807+ if cv == candidates.Prev.val && offset <= maxMatchOffset {
808+ candidateAlt = candidates.Prev
809+ }
810+ continue
811+ }
812+ } else {
813+ // We only check if value mismatches.
814+ // Offset will always be invalid in other cases.
815+ candidate = candidates.Prev
816+ if cv == candidate.val {
817+ offset := s - (candidate.offset - e.cur)
818+ if offset <= maxMatchOffset {
819+ continue
820+ }
821+ }
822+ }
823+ cv = uint32(x >> 8)
824+ nextHash = hash(cv)
825+ s++
826+ break
827+ }
828+ }
829+
830+emitRemainder:
831+ if int(nextEmit) < len(src) {
832+ emitLiteral(dst, src[nextEmit:])
833+ }
834+ e.cur += int32(len(src))
835+ e.prev = e.prev[:len(src)]
836+ copy(e.prev, src)
837+}
838+
839+func (e *snappyGen) matchlen(s, t int32, src []byte) int32 {
840+ s1 := int(s) + maxMatchLength - 4
841+ if s1 > len(src) {
842+ s1 = len(src)
843+ }
844+
845+ // If we are inside the current block
846+ if t >= 0 {
847+ b := src[t:]
848+ a := src[s:s1]
849+ b = b[:len(a)]
850+ // Extend the match to be as long as possible.
851+ for i := range a {
852+ if a[i] != b[i] {
853+ return int32(i)
854+ }
855+ }
856+ return int32(len(a))
857+ }
858+
859+ // We found a match in the previous block.
860+ tp := int32(len(e.prev)) + t
861+ if tp < 0 {
862+ return 0
863+ }
864+
865+ // Extend the match to be as long as possible.
866+ a := src[s:s1]
867+ b := e.prev[tp:]
868+ if len(b) > len(a) {
869+ b = b[:len(a)]
870+ }
871+ a = a[:len(b)]
872+ for i := range b {
873+ if a[i] != b[i] {
874+ return int32(i)
875+ }
876+ }
877+
878+ // If we reached our limit, we matched everything we are
879+ // allowed to in the previous block and we return.
880+ n := int32(len(b))
881+ if int(s+n) == s1 {
882+ return n
883+ }
884+
885+ // Continue looking for more matches in the current block.
886+ a = src[s+n : s1]
887+ b = src[:len(a)]
888+ for i := range a {
889+ if a[i] != b[i] {
890+ return int32(i) + n
891+ }
892+ }
893+ return int32(len(a)) + n
894+}
895+
896+// Reset the encoding table.
897+func (e *snappyGen) Reset() {
898+ e.prev = e.prev[:0]
899+ e.cur += maxMatchOffset
900+}
vendor/github.com/klauspost/compress/flate/token.go+115−0
@@ -0,0 +1,115 @@
1+// Copyright 2009 The Go Authors. All rights reserved.
2+// Use of this source code is governed by a BSD-style
3+// license that can be found in the LICENSE file.
4+
5+package flate
6+
7+import "fmt"
8+
9+const (
10+ // 2 bits: type 0 = literal 1=EOF 2=Match 3=Unused
11+ // 8 bits: xlength = length - MIN_MATCH_LENGTH
12+ // 22 bits xoffset = offset - MIN_OFFSET_SIZE, or literal
13+ lengthShift = 22
14+ offsetMask = 1<<lengthShift - 1
15+ typeMask = 3 << 30
16+ literalType = 0 << 30
17+ matchType = 1 << 30
18+)
19+
20+// The length code for length X (MIN_MATCH_LENGTH <= X <= MAX_MATCH_LENGTH)
21+// is lengthCodes[length - MIN_MATCH_LENGTH]
22+var lengthCodes = [...]uint32{
23+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 8,
24+ 9, 9, 10, 10, 11, 11, 12, 12, 12, 12,
25+ 13, 13, 13, 13, 14, 14, 14, 14, 15, 15,
26+ 15, 15, 16, 16, 16, 16, 16, 16, 16, 16,
27+ 17, 17, 17, 17, 17, 17, 17, 17, 18, 18,
28+ 18, 18, 18, 18, 18, 18, 19, 19, 19, 19,
29+ 19, 19, 19, 19, 20, 20, 20, 20, 20, 20,
30+ 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
31+ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
32+ 21, 21, 21, 21, 21, 21, 22, 22, 22, 22,
33+ 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
34+ 22, 22, 23, 23, 23, 23, 23, 23, 23, 23,
35+ 23, 23, 23, 23, 23, 23, 23, 23, 24, 24,
36+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
37+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
38+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
39+ 25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
40+ 25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
41+ 25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
42+ 25, 25, 26, 26, 26, 26, 26, 26, 26, 26,
43+ 26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
44+ 26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
45+ 26, 26, 26, 26, 27, 27, 27, 27, 27, 27,
46+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
47+ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
48+ 27, 27, 27, 27, 27, 28,
49+}
50+
51+var offsetCodes = [...]uint32{
52+ 0, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7,
53+ 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9,
54+ 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
55+ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
56+ 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
57+ 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
58+ 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
59+ 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
60+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
61+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
62+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
63+ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
64+ 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
65+ 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
66+ 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
67+ 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
68+}
69+
70+type token uint32
71+
72+type tokens struct {
73+ tokens [maxStoreBlockSize + 1]token
74+ n uint16 // Must be able to contain maxStoreBlockSize
75+}
76+
77+// Convert a literal into a literal token.
78+func literalToken(literal uint32) token { return token(literalType + literal) }
79+
80+// Convert a < xlength, xoffset > pair into a match token.
81+func matchToken(xlength uint32, xoffset uint32) token {
82+ return token(matchType + xlength<<lengthShift + xoffset)
83+}
84+
85+func matchTokend(xlength uint32, xoffset uint32) token {
86+ if xlength > maxMatchLength || xoffset > maxMatchOffset {
87+ panic(fmt.Sprintf("Invalid match: len: %d, offset: %d\n", xlength, xoffset))
88+ return token(matchType)
89+ }
90+ return token(matchType + xlength<<lengthShift + xoffset)
91+}
92+
93+// Returns the type of a token
94+func (t token) typ() uint32 { return uint32(t) & typeMask }
95+
96+// Returns the literal of a literal token
97+func (t token) literal() uint32 { return uint32(t - literalType) }
98+
99+// Returns the extra offset of a match token
100+func (t token) offset() uint32 { return uint32(t) & offsetMask }
101+
102+func (t token) length() uint32 { return uint32((t - matchType) >> lengthShift) }
103+
104+func lengthCode(len uint32) uint32 { return lengthCodes[len] }
105+
106+// Returns the offset code corresponding to a specific offset
107+func offsetCode(off uint32) uint32 {
108+ if off < uint32(len(offsetCodes)) {
109+ return offsetCodes[off]
110+ } else if off>>7 < uint32(len(offsetCodes)) {
111+ return offsetCodes[off>>7] + 14
112+ } else {
113+ return offsetCodes[off>>14] + 28
114+ }
115+}
vendor/github.com/klauspost/pgzip/GO_LICENSE+27−0
@@ -0,0 +1,27 @@
1+Copyright (c) 2012 The Go Authors. All rights reserved.
2+
3+Redistribution and use in source and binary forms, with or without
4+modification, are permitted provided that the following conditions are
5+met:
6+
7+ * Redistributions of source code must retain the above copyright
8+notice, this list of conditions and the following disclaimer.
9+ * Redistributions in binary form must reproduce the above
10+copyright notice, this list of conditions and the following disclaimer
11+in the documentation and/or other materials provided with the
12+distribution.
13+ * Neither the name of Google Inc. nor the names of its
14+contributors may be used to endorse or promote products derived from
15+this software without specific prior written permission.
16+
17+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
vendor/github.com/klauspost/pgzip/LICENSE+22−0
@@ -0,0 +1,22 @@
1+The MIT License (MIT)
2+
3+Copyright (c) 2014 Klaus Post
4+
5+Permission is hereby granted, free of charge, to any person obtaining a copy
6+of this software and associated documentation files (the "Software"), to deal
7+in the Software without restriction, including without limitation the rights
8+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+copies of the Software, and to permit persons to whom the Software is
10+furnished to do so, subject to the following conditions:
11+
12+The above copyright notice and this permission notice shall be included in all
13+copies or substantial portions of the Software.
14+
15+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+SOFTWARE.
22+
vendor/github.com/klauspost/pgzip/README.md+136−0
@@ -0,0 +1,136 @@
1+pgzip
2+=====
3+
4+Go parallel gzip compression/decompression. This is a fully gzip compatible drop in replacement for "compress/gzip".
5+
6+This will split compression into blocks that are compressed in parallel.
7+This can be useful for compressing big amounts of data. The output is a standard gzip file.
8+
9+The gzip decompression is modified so it decompresses ahead of the current reader.
10+This means that reads will be non-blocking if the decompressor can keep ahead of your code reading from it.
11+CRC calculation also takes place in a separate goroutine.
12+
13+You should only use this if you are (de)compressing big amounts of data,
14+say **more than 1MB** at the time, otherwise you will not see any benefit,
15+and it will likely be faster to use the internal gzip library
16+or [this package](https://github.com/klauspost/compress).
17+
18+It is important to note that this library creates and reads *standard gzip files*.
19+You do not have to match the compressor/decompressor to get the described speedups,
20+and the gzip files are fully compatible with other gzip readers/writers.
21+
22+A golang variant of this is [bgzf](https://godoc.org/github.com/biogo/hts/bgzf),
23+which has the same feature, as well as seeking in the resulting file.
24+The only drawback is a slightly bigger overhead compared to this and pure gzip.
25+See a comparison below.
26+
27+[![GoDoc][1]][2] [![Build Status][3]][4]
28+
29+[1]: https://godoc.org/github.com/klauspost/pgzip?status.svg
30+[2]: https://godoc.org/github.com/klauspost/pgzip
31+[3]: https://travis-ci.org/klauspost/pgzip.svg
32+[4]: https://travis-ci.org/klauspost/pgzip
33+
34+Installation
35+====
36+```go get github.com/klauspost/pgzip/...```
37+
38+You might need to get/update the dependencies:
39+
40+```
41+go get -u github.com/klauspost/compress
42+go get -u github.com/klauspost/crc32
43+```
44+
45+Usage
46+====
47+[Godoc Doumentation](https://godoc.org/github.com/klauspost/pgzip)
48+
49+To use as a replacement for gzip, exchange
50+
51+```import "compress/gzip"```
52+with
53+```import gzip "github.com/klauspost/pgzip"```.
54+
55+# Changes
56+
57+* Oct 6, 2016: Fixed an issue if the destination writer returned an error.
58+* Oct 6, 2016: Better buffer reuse, should now generate less garbage.
59+* Oct 6, 2016: Output does not change based on write sizes.
60+* Dec 8, 2015: Decoder now supports the io.WriterTo interface, giving a speedup and less GC pressure.
61+* Oct 9, 2015: Reduced allocations by ~35 by using sync.Pool. ~15% overall speedup.
62+
63+Changes in [github.com/klauspost/compress](https://github.com/klauspost/compress#changelog) are also carried over, so see that for more changes.
64+
65+## Compression
66+The simplest way to use this is to simply do the same as you would when using [compress/gzip](http://golang.org/pkg/compress/gzip).
67+
68+To change the block size, use the added (*pgzip.Writer).SetConcurrency(blockSize, blocks int) function. With this you can control the approximate size of your blocks, as well as how many you want to be processing in parallel. Default values for this is SetConcurrency(250000, 16), meaning blocks are split at 250000 bytes and up to 16 blocks can be processing at once before the writer blocks.
69+
70+
71+Example:
72+```
73+var b bytes.Buffer
74+w := gzip.NewWriter(&b)
75+w.SetConcurrency(100000, 10)
76+w.Write([]byte("hello, world\n"))
77+w.Close()
78+```
79+
80+To get any performance gains, you should at least be compressing more than 1 megabyte of data at the time.
81+
82+You should at least have a block size of 100k and at least a number of blocks that match the number of cores your would like to utilize, but about twice the number of blocks would be the best.
83+
84+Another side effect of this is, that it is likely to speed up your other code, since writes to the compressor only blocks if the compressor is already compressing the number of blocks you have specified. This also means you don't have worry about buffering input to the compressor.
85+
86+## Decompression
87+
88+Decompression works similar to compression. That means that you simply call pgzip the same way as you would call [compress/gzip](http://golang.org/pkg/compress/gzip).
89+
90+The only difference is that if you want to specify your own readahead, you have to use `pgzip.NewReaderN(r io.Reader, blockSize, blocks int)` to get a reader with your custom blocksizes. The `blockSize` is the size of each block decoded, and `blocks` is the maximum number of blocks that is decoded ahead.
91+
92+See [Example on playground](http://play.golang.org/p/uHv1B5NbDh)
93+
94+Performance
95+====
96+## Compression
97+
98+See my blog post in [Benchmarks of Golang Gzip](https://blog.klauspost.com/go-gzipdeflate-benchmarks/).
99+
100+Compression cost is usually about 0.2% with default settings with a block size of 250k.
101+
102+Example with GOMAXPROC set to 8 (quad core with 8 hyperthreads)
103+
104+Content is [Matt Mahoneys 10GB corpus](http://mattmahoney.net/dc/10gb.html). Compression level 6.
105+
106+Compressor | MB/sec | speedup | size | size overhead (lower=better)
107+------------|----------|---------|------|---------
108+[gzip](http://golang.org/pkg/compress/gzip) (golang) | 7.21MB/s | 1.0x | 4786608902 | 0%
109+[gzip](http://github.com/klauspost/compress/gzip) (klauspost) | 10.98MB/s | 1.52x | 4781331645 | -0.11%
110+[pgzip](https://github.com/klauspost/pgzip) (klauspost) | 50.76MB/s|7.04x | 4784121440 | -0.052%
111+[bgzf](https://godoc.org/github.com/biogo/hts/bgzf) (biogo) | 38.65MB/s | 5.36x | 4924899484 | 2.889%
112+[pargzip](https://godoc.org/github.com/golang/build/pargzip) (builder) | 32.00MB/s | 4.44x | 4791226567 | 0.096%
113+
114+pgzip also contains a [linear time compression](https://github.com/klauspost/compress#linear-time-compression) mode, that will allow compression at ~150MB per core per second, independent of the content.
115+
116+See the [complete sheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing) for different content types and compression settings.
117+
118+## Decompression
119+
120+The decompression speedup is there because it allows you to do other work while the decompression is taking place.
121+
122+In the example above, the numbers are as follows on a 4 CPU machine:
123+
124+Decompressor | Time | Speedup
125+-------------|------|--------
126+[gzip](http://golang.org/pkg/compress/gzip) (golang) | 1m28.85s | 0%
127+[pgzip](https://github.com/klauspost/pgzip) (golang) | 43.48s | 104%
128+
129+But wait, since gzip decompression is inherently singlethreaded (aside from CRC calculation) how can it be more than 100% faster? Because pgzip due to its design also acts as a buffer. When using unbuffered gzip, you are also waiting for io when you are decompressing. If the gzip decoder can keep up, it will always have data ready for your reader, and you will not be waiting for input to the gzip decompressor to complete.
130+
131+This is pretty much an optimal situation for pgzip, but it reflects most common usecases for CPU intensive gzip usage.
132+
133+I haven't included [bgzf](https://godoc.org/github.com/biogo/hts/bgzf) in this comparison, since it only can decompress files created by a compatible encoder, and therefore cannot be considered a generic gzip decompressor. But if you are able to compress your files with a bgzf compatible program, you can expect it to scale beyond 100%.
134+
135+# License
136+This contains large portions of code from the go repository - see GO_LICENSE for more information. The changes are released under MIT License. See LICENSE for more information.
vendor/github.com/klauspost/pgzip/circle.yml+7−0
@@ -0,0 +1,7 @@
1+test:
2+ pre:
3+ - go vet ./...
4+
5+ override:
6+ - go test -v -cpu=1,2,4 .
7+ - go test -v -cpu=2 -race -short .
vendor/github.com/klauspost/pgzip/gunzip.go+573−0
@@ -0,0 +1,573 @@
1+// Copyright 2009 The Go Authors. All rights reserved.
2+// Use of this source code is governed by a BSD-style
3+// license that can be found in the LICENSE file.
4+
5+// Package pgzip implements reading and writing of gzip format compressed files,
6+// as specified in RFC 1952.
7+//
8+// This is a drop in replacement for "compress/gzip".
9+// This will split compression into blocks that are compressed in parallel.
10+// This can be useful for compressing big amounts of data.
11+// The gzip decompression has not been modified, but remains in the package,
12+// so you can use it as a complete replacement for "compress/gzip".
13+//
14+// See more at https://github.com/klauspost/pgzip
15+package pgzip
16+
17+import (
18+ "bufio"
19+ "errors"
20+ "hash"
21+ "io"
22+ "sync"
23+ "time"
24+
25+ "github.com/klauspost/compress/flate"
26+ "github.com/klauspost/crc32"
27+)
28+
29+const (
30+ gzipID1 = 0x1f
31+ gzipID2 = 0x8b
32+ gzipDeflate = 8
33+ flagText = 1 << 0
34+ flagHdrCrc = 1 << 1
35+ flagExtra = 1 << 2
36+ flagName = 1 << 3
37+ flagComment = 1 << 4
38+)
39+
40+func makeReader(r io.Reader) flate.Reader {
41+ if rr, ok := r.(flate.Reader); ok {
42+ return rr
43+ }
44+ return bufio.NewReader(r)
45+}
46+
47+var (
48+ // ErrChecksum is returned when reading GZIP data that has an invalid checksum.
49+ ErrChecksum = errors.New("gzip: invalid checksum")
50+ // ErrHeader is returned when reading GZIP data that has an invalid header.
51+ ErrHeader = errors.New("gzip: invalid header")
52+)
53+
54+// The gzip file stores a header giving metadata about the compressed file.
55+// That header is exposed as the fields of the Writer and Reader structs.
56+type Header struct {
57+ Comment string // comment
58+ Extra []byte // "extra data"
59+ ModTime time.Time // modification time
60+ Name string // file name
61+ OS byte // operating system type
62+}
63+
64+// A Reader is an io.Reader that can be read to retrieve
65+// uncompressed data from a gzip-format compressed file.
66+//
67+// In general, a gzip file can be a concatenation of gzip files,
68+// each with its own header. Reads from the Reader
69+// return the concatenation of the uncompressed data of each.
70+// Only the first header is recorded in the Reader fields.
71+//
72+// Gzip files store a length and checksum of the uncompressed data.
73+// The Reader will return a ErrChecksum when Read
74+// reaches the end of the uncompressed data if it does not
75+// have the expected length or checksum. Clients should treat data
76+// returned by Read as tentative until they receive the io.EOF
77+// marking the end of the data.
78+type Reader struct {
79+ Header
80+ r flate.Reader
81+ decompressor io.ReadCloser
82+ digest hash.Hash32
83+ size uint32
84+ flg byte
85+ buf [512]byte
86+ err error
87+ closeErr chan error
88+ multistream bool
89+
90+ readAhead chan read
91+ roff int // read offset
92+ current []byte
93+ closeReader chan struct{}
94+ lastBlock bool
95+ blockSize int
96+ blocks int
97+
98+ activeRA bool // Indication if readahead is active
99+ mu sync.Mutex // Lock for above
100+
101+ blockPool chan []byte
102+}
103+
104+type read struct {
105+ b []byte
106+ err error
107+}
108+
109+// NewReader creates a new Reader reading the given reader.
110+// The implementation buffers input and may read more data than necessary from r.
111+// It is the caller's responsibility to call Close on the Reader when done.
112+func NewReader(r io.Reader) (*Reader, error) {
113+ z := new(Reader)
114+ z.blocks = defaultBlocks
115+ z.blockSize = defaultBlockSize
116+ z.r = makeReader(r)
117+ z.digest = crc32.NewIEEE()
118+ z.multistream = true
119+ z.blockPool = make(chan []byte, z.blocks)
120+ for i := 0; i < z.blocks; i++ {
121+ z.blockPool <- make([]byte, z.blockSize)
122+ }
123+ if err := z.readHeader(true); err != nil {
124+ return nil, err
125+ }
126+ return z, nil
127+}
128+
129+// NewReaderN creates a new Reader reading the given reader.
130+// The implementation buffers input and may read more data than necessary from r.
131+// It is the caller's responsibility to call Close on the Reader when done.
132+//
133+// With this you can control the approximate size of your blocks,
134+// as well as how many blocks you want to have prefetched.
135+//
136+// Default values for this is blockSize = 250000, blocks = 16,
137+// meaning up to 16 blocks of maximum 250000 bytes will be
138+// prefetched.
139+func NewReaderN(r io.Reader, blockSize, blocks int) (*Reader, error) {
140+ z := new(Reader)
141+ z.blocks = blocks
142+ z.blockSize = blockSize
143+ z.r = makeReader(r)
144+ z.digest = crc32.NewIEEE()
145+ z.multistream = true
146+
147+ // Account for too small values
148+ if z.blocks <= 0 {
149+ z.blocks = defaultBlocks
150+ }
151+ if z.blockSize <= 512 {
152+ z.blockSize = defaultBlockSize
153+ }
154+ z.blockPool = make(chan []byte, z.blocks)
155+ for i := 0; i < z.blocks; i++ {
156+ z.blockPool <- make([]byte, z.blockSize)
157+ }
158+ if err := z.readHeader(true); err != nil {
159+ return nil, err
160+ }
161+ return z, nil
162+}
163+
164+// Reset discards the Reader z's state and makes it equivalent to the
165+// result of its original state from NewReader, but reading from r instead.
166+// This permits reusing a Reader rather than allocating a new one.
167+func (z *Reader) Reset(r io.Reader) error {
168+ z.killReadAhead()
169+ z.r = makeReader(r)
170+ z.digest = crc32.NewIEEE()
171+ z.size = 0
172+ z.err = nil
173+ z.multistream = true
174+
175+ // Account for uninitialized values
176+ if z.blocks <= 0 {
177+ z.blocks = defaultBlocks
178+ }
179+ if z.blockSize <= 512 {
180+ z.blockSize = defaultBlockSize
181+ }
182+
183+ if z.blockPool == nil {
184+ z.blockPool = make(chan []byte, z.blocks)
185+ for i := 0; i < z.blocks; i++ {
186+ z.blockPool <- make([]byte, z.blockSize)
187+ }
188+ }
189+
190+ return z.readHeader(true)
191+}
192+
193+// Multistream controls whether the reader supports multistream files.
194+//
195+// If enabled (the default), the Reader expects the input to be a sequence
196+// of individually gzipped data streams, each with its own header and
197+// trailer, ending at EOF. The effect is that the concatenation of a sequence
198+// of gzipped files is treated as equivalent to the gzip of the concatenation
199+// of the sequence. This is standard behavior for gzip readers.
200+//
201+// Calling Multistream(false) disables this behavior; disabling the behavior
202+// can be useful when reading file formats that distinguish individual gzip
203+// data streams or mix gzip data streams with other data streams.
204+// In this mode, when the Reader reaches the end of the data stream,
205+// Read returns io.EOF. If the underlying reader implements io.ByteReader,
206+// it will be left positioned just after the gzip stream.
207+// To start the next stream, call z.Reset(r) followed by z.Multistream(false).
208+// If there is no next stream, z.Reset(r) will return io.EOF.
209+func (z *Reader) Multistream(ok bool) {
210+ z.multistream = ok
211+}
212+
213+// GZIP (RFC 1952) is little-endian, unlike ZLIB (RFC 1950).
214+func get4(p []byte) uint32 {
215+ return uint32(p[0]) | uint32(p[1])<<8 | uint32(p[2])<<16 | uint32(p[3])<<24
216+}
217+
218+func (z *Reader) readString() (string, error) {
219+ var err error
220+ needconv := false
221+ for i := 0; ; i++ {
222+ if i >= len(z.buf) {
223+ return "", ErrHeader
224+ }
225+ z.buf[i], err = z.r.ReadByte()
226+ if err != nil {
227+ return "", err
228+ }
229+ if z.buf[i] > 0x7f {
230+ needconv = true
231+ }
232+ if z.buf[i] == 0 {
233+ // GZIP (RFC 1952) specifies that strings are NUL-terminated ISO 8859-1 (Latin-1).
234+ if needconv {
235+ s := make([]rune, 0, i)
236+ for _, v := range z.buf[0:i] {
237+ s = append(s, rune(v))
238+ }
239+ return string(s), nil
240+ }
241+ return string(z.buf[0:i]), nil
242+ }
243+ }
244+}
245+
246+func (z *Reader) read2() (uint32, error) {
247+ _, err := io.ReadFull(z.r, z.buf[0:2])
248+ if err != nil {
249+ return 0, err
250+ }
251+ return uint32(z.buf[0]) | uint32(z.buf[1])<<8, nil
252+}
253+
254+func (z *Reader) readHeader(save bool) error {
255+ z.killReadAhead()
256+
257+ _, err := io.ReadFull(z.r, z.buf[0:10])
258+ if err != nil {
259+ return err
260+ }
261+ if z.buf[0] != gzipID1 || z.buf[1] != gzipID2 || z.buf[2] != gzipDeflate {
262+ return ErrHeader
263+ }
264+ z.flg = z.buf[3]
265+ if save {
266+ z.ModTime = time.Unix(int64(get4(z.buf[4:8])), 0)
267+ // z.buf[8] is xfl, ignored
268+ z.OS = z.buf[9]
269+ }
270+ z.digest.Reset()
271+ z.digest.Write(z.buf[0:10])
272+
273+ if z.flg&flagExtra != 0 {
274+ n, err := z.read2()
275+ if err != nil {
276+ return err
277+ }
278+ data := make([]byte, n)
279+ if _, err = io.ReadFull(z.r, data); err != nil {
280+ return err
281+ }
282+ if save {
283+ z.Extra = data
284+ }
285+ }
286+
287+ var s string
288+ if z.flg&flagName != 0 {
289+ if s, err = z.readString(); err != nil {
290+ return err
291+ }
292+ if save {
293+ z.Name = s
294+ }
295+ }
296+
297+ if z.flg&flagComment != 0 {
298+ if s, err = z.readString(); err != nil {
299+ return err
300+ }
301+ if save {
302+ z.Comment = s
303+ }
304+ }
305+
306+ if z.flg&flagHdrCrc != 0 {
307+ n, err := z.read2()
308+ if err != nil {
309+ return err
310+ }
311+ sum := z.digest.Sum32() & 0xFFFF
312+ if n != sum {
313+ return ErrHeader
314+ }
315+ }
316+
317+ z.digest.Reset()
318+ z.decompressor = flate.NewReader(z.r)
319+ z.doReadAhead()
320+ return nil
321+}
322+
323+func (z *Reader) killReadAhead() error {
324+ z.mu.Lock()
325+ defer z.mu.Unlock()
326+ if z.activeRA {
327+ if z.closeReader != nil {
328+ close(z.closeReader)
329+ }
330+
331+ // Wait for decompressor to be closed and return error, if any.
332+ e, ok := <-z.closeErr
333+ z.activeRA = false
334+ if !ok {
335+ // Channel is closed, so if there was any error it has already been returned.
336+ return nil
337+ }
338+ return e
339+ }
340+ return nil
341+}
342+
343+// Starts readahead.
344+// Will return on error (including io.EOF)
345+// or when z.closeReader is closed.
346+func (z *Reader) doReadAhead() {
347+ z.mu.Lock()
348+ defer z.mu.Unlock()
349+ z.activeRA = true
350+
351+ if z.blocks <= 0 {
352+ z.blocks = defaultBlocks
353+ }
354+ if z.blockSize <= 512 {
355+ z.blockSize = defaultBlockSize
356+ }
357+ ra := make(chan read, z.blocks)
358+ z.readAhead = ra
359+ closeReader := make(chan struct{}, 0)
360+ z.closeReader = closeReader
361+ z.lastBlock = false
362+ closeErr := make(chan error, 1)
363+ z.closeErr = closeErr
364+ z.size = 0
365+ z.roff = 0
366+ z.current = nil
367+ decomp := z.decompressor
368+
369+ go func() {
370+ defer func() {
371+ closeErr <- decomp.Close()
372+ close(closeErr)
373+ close(ra)
374+ }()
375+
376+ // We hold a local reference to digest, since
377+ // it way be changed by reset.
378+ digest := z.digest
379+ var wg sync.WaitGroup
380+ for {
381+ var buf []byte
382+ select {
383+ case buf = <-z.blockPool:
384+ case <-closeReader:
385+ return
386+ }
387+ buf = buf[0:z.blockSize]
388+ // Try to fill the buffer
389+ n, err := io.ReadFull(decomp, buf)
390+ if err == io.ErrUnexpectedEOF {
391+ if n > 0 {
392+ err = nil
393+ } else {
394+ // If we got zero bytes, we need to establish if
395+ // we reached end of stream or truncated stream.
396+ _, err = decomp.Read([]byte{})
397+ if err == io.EOF {
398+ err = nil
399+ }
400+ }
401+ }
402+ if n < len(buf) {
403+ buf = buf[0:n]
404+ }
405+ wg.Wait()
406+ wg.Add(1)
407+ go func() {
408+ digest.Write(buf)
409+ wg.Done()
410+ }()
411+ z.size += uint32(n)
412+
413+ // If we return any error, out digest must be ready
414+ if err != nil {
415+ wg.Wait()
416+ }
417+ select {
418+ case z.readAhead <- read{b: buf, err: err}:
419+ case <-closeReader:
420+ // Sent on close, we don't care about the next results
421+ return
422+ }
423+ if err != nil {
424+ return
425+ }
426+ }
427+ }()
428+}
429+
430+func (z *Reader) Read(p []byte) (n int, err error) {
431+ if z.err != nil {
432+ return 0, z.err
433+ }
434+ if len(p) == 0 {
435+ return 0, nil
436+ }
437+
438+ for {
439+ if len(z.current) == 0 && !z.lastBlock {
440+ read := <-z.readAhead
441+
442+ if read.err != nil {
443+ // If not nil, the reader will have exited
444+ z.closeReader = nil
445+
446+ if read.err != io.EOF {
447+ z.err = read.err
448+ return
449+ }
450+ if read.err == io.EOF {
451+ z.lastBlock = true
452+ err = nil
453+ }
454+ }
455+ z.current = read.b
456+ z.roff = 0
457+ }
458+ avail := z.current[z.roff:]
459+ if len(p) >= len(avail) {
460+ // If len(p) >= len(current), return all content of current
461+ n = copy(p, avail)
462+ z.blockPool <- z.current
463+ z.current = nil
464+ if z.lastBlock {
465+ err = io.EOF
466+ break
467+ }
468+ } else {
469+ // We copy as much as there is space for
470+ n = copy(p, avail)
471+ z.roff += n
472+ }
473+ return
474+ }
475+
476+ // Finished file; check checksum + size.
477+ if _, err := io.ReadFull(z.r, z.buf[0:8]); err != nil {
478+ z.err = err
479+ return 0, err
480+ }
481+ crc32, isize := get4(z.buf[0:4]), get4(z.buf[4:8])
482+ sum := z.digest.Sum32()
483+ if sum != crc32 || isize != z.size {
484+ z.err = ErrChecksum
485+ return 0, z.err
486+ }
487+
488+ // File is ok; should we attempt reading one more?
489+ if !z.multistream {
490+ return 0, io.EOF
491+ }
492+
493+ // Is there another?
494+ if err = z.readHeader(false); err != nil {
495+ z.err = err
496+ return
497+ }
498+
499+ // Yes. Reset and read from it.
500+ return z.Read(p)
501+}
502+
503+func (z *Reader) WriteTo(w io.Writer) (n int64, err error) {
504+ total := int64(0)
505+ for {
506+ if z.err != nil {
507+ return total, z.err
508+ }
509+ // We write both to output and digest.
510+ for {
511+ // Read from input
512+ read := <-z.readAhead
513+ if read.err != nil {
514+ // If not nil, the reader will have exited
515+ z.closeReader = nil
516+
517+ if read.err != io.EOF {
518+ z.err = read.err
519+ return total, z.err
520+ }
521+ if read.err == io.EOF {
522+ z.lastBlock = true
523+ err = nil
524+ }
525+ }
526+ // Write what we got
527+ n, err := w.Write(read.b)
528+ if n != len(read.b) {
529+ return total, io.ErrShortWrite
530+ }
531+ total += int64(n)
532+ if err != nil {
533+ return total, err
534+ }
535+ // Put block back
536+ z.blockPool <- read.b
537+ if z.lastBlock {
538+ break
539+ }
540+ }
541+
542+ // Finished file; check checksum + size.
543+ if _, err := io.ReadFull(z.r, z.buf[0:8]); err != nil {
544+ z.err = err
545+ return total, err
546+ }
547+ crc32, isize := get4(z.buf[0:4]), get4(z.buf[4:8])
548+ sum := z.digest.Sum32()
549+ if sum != crc32 || isize != z.size {
550+ z.err = ErrChecksum
551+ return total, z.err
552+ }
553+ // File is ok; should we attempt reading one more?
554+ if !z.multistream {
555+ return total, nil
556+ }
557+
558+ // Is there another?
559+ err = z.readHeader(false)
560+ if err == io.EOF {
561+ return total, nil
562+ }
563+ if err != nil {
564+ z.err = err
565+ return total, err
566+ }
567+ }
568+}
569+
570+// Close closes the Reader. It does not close the underlying io.Reader.
571+func (z *Reader) Close() error {
572+ return z.killReadAhead()
573+}
vendor/github.com/klauspost/pgzip/gzip.go+501−0
@@ -0,0 +1,501 @@
1+// Copyright 2010 The Go Authors. All rights reserved.
2+// Use of this source code is governed by a BSD-style
3+// license that can be found in the LICENSE file.
4+
5+package pgzip
6+
7+import (
8+ "bytes"
9+ "errors"
10+ "fmt"
11+ "hash"
12+ "io"
13+ "sync"
14+ "time"
15+
16+ "github.com/klauspost/compress/flate"
17+ "github.com/klauspost/crc32"
18+)
19+
20+const (
21+ defaultBlockSize = 256 << 10
22+ tailSize = 16384
23+ defaultBlocks = 16
24+)
25+
26+// These constants are copied from the flate package, so that code that imports
27+// "compress/gzip" does not also have to import "compress/flate".
28+const (
29+ NoCompression = flate.NoCompression
30+ BestSpeed = flate.BestSpeed
31+ BestCompression = flate.BestCompression
32+ DefaultCompression = flate.DefaultCompression
33+ ConstantCompression = flate.ConstantCompression
34+ HuffmanOnly = flate.HuffmanOnly
35+)
36+
37+// A Writer is an io.WriteCloser.
38+// Writes to a Writer are compressed and written to w.
39+type Writer struct {
40+ Header
41+ w io.Writer
42+ level int
43+ wroteHeader bool
44+ blockSize int
45+ blocks int
46+ currentBuffer []byte
47+ prevTail []byte
48+ digest hash.Hash32
49+ size int
50+ closed bool
51+ buf [10]byte
52+ errMu sync.RWMutex
53+ err error
54+ pushedErr chan struct{}
55+ results chan result
56+ dictFlatePool sync.Pool
57+ dstPool sync.Pool
58+ wg sync.WaitGroup
59+}
60+
61+type result struct {
62+ result chan []byte
63+ notifyWritten chan struct{}
64+}
65+
66+// Use SetConcurrency to finetune the concurrency level if needed.
67+//
68+// With this you can control the approximate size of your blocks,
69+// as well as how many you want to be processing in parallel.
70+//
71+// Default values for this is SetConcurrency(250000, 16),
72+// meaning blocks are split at 250000 bytes and up to 16 blocks
73+// can be processing at once before the writer blocks.
74+func (z *Writer) SetConcurrency(blockSize, blocks int) error {
75+ if blockSize <= tailSize {
76+ return fmt.Errorf("gzip: block size cannot be less than or equal to %d", tailSize)
77+ }
78+ if blocks <= 0 {
79+ return errors.New("gzip: blocks cannot be zero or less")
80+ }
81+ if blockSize == z.blockSize && blocks == z.blocks {
82+ return nil
83+ }
84+ z.blockSize = blockSize
85+ z.results = make(chan result, blocks)
86+ z.blocks = blocks
87+ z.dstPool = sync.Pool{New: func() interface{} { return make([]byte, 0, blockSize+(blockSize)>>4) }}
88+ return nil
89+}
90+
91+// NewWriter returns a new Writer.
92+// Writes to the returned writer are compressed and written to w.
93+//
94+// It is the caller's responsibility to call Close on the WriteCloser when done.
95+// Writes may be buffered and not flushed until Close.
96+//
97+// Callers that wish to set the fields in Writer.Header must do so before
98+// the first call to Write or Close. The Comment and Name header fields are
99+// UTF-8 strings in Go, but the underlying format requires NUL-terminated ISO
100+// 8859-1 (Latin-1). NUL or non-Latin-1 runes in those strings will lead to an
101+// error on Write.
102+func NewWriter(w io.Writer) *Writer {
103+ z, _ := NewWriterLevel(w, DefaultCompression)
104+ return z
105+}
106+
107+// NewWriterLevel is like NewWriter but specifies the compression level instead
108+// of assuming DefaultCompression.
109+//
110+// The compression level can be DefaultCompression, NoCompression, or any
111+// integer value between BestSpeed and BestCompression inclusive. The error
112+// returned will be nil if the level is valid.
113+func NewWriterLevel(w io.Writer, level int) (*Writer, error) {
114+ if level < ConstantCompression || level > BestCompression {
115+ return nil, fmt.Errorf("gzip: invalid compression level: %d", level)
116+ }
117+ z := new(Writer)
118+ z.SetConcurrency(defaultBlockSize, defaultBlocks)
119+ z.init(w, level)
120+ return z, nil
121+}
122+
123+// This function must be used by goroutines to set an
124+// error condition, since z.err access is restricted
125+// to the callers goruotine.
126+func (z *Writer) pushError(err error) {
127+ z.errMu.Lock()
128+ if z.err != nil {
129+ z.errMu.Unlock()
130+ return
131+ }
132+ z.err = err
133+ close(z.pushedErr)
134+ z.errMu.Unlock()
135+}
136+
137+func (z *Writer) init(w io.Writer, level int) {
138+ z.wg.Wait()
139+ digest := z.digest
140+ if digest != nil {
141+ digest.Reset()
142+ } else {
143+ digest = crc32.NewIEEE()
144+ }
145+ z.Header = Header{OS: 255}
146+ z.w = w
147+ z.level = level
148+ z.digest = digest
149+ z.pushedErr = make(chan struct{}, 0)
150+ z.results = make(chan result, z.blocks)
151+ z.err = nil
152+ z.closed = false
153+ z.Comment = ""
154+ z.Extra = nil
155+ z.ModTime = time.Time{}
156+ z.wroteHeader = false
157+ z.currentBuffer = nil
158+ z.buf = [10]byte{}
159+ z.prevTail = nil
160+ z.size = 0
161+ if z.dictFlatePool.New == nil {
162+ z.dictFlatePool.New = func() interface{} {
163+ f, _ := flate.NewWriterDict(w, level, nil)
164+ return f
165+ }
166+ }
167+}
168+
169+// Reset discards the Writer z's state and makes it equivalent to the
170+// result of its original state from NewWriter or NewWriterLevel, but
171+// writing to w instead. This permits reusing a Writer rather than
172+// allocating a new one.
173+func (z *Writer) Reset(w io.Writer) {
174+ if z.results != nil && !z.closed {
175+ close(z.results)
176+ }
177+ z.SetConcurrency(defaultBlockSize, defaultBlocks)
178+ z.init(w, z.level)
179+}
180+
181+// GZIP (RFC 1952) is little-endian, unlike ZLIB (RFC 1950).
182+func put2(p []byte, v uint16) {
183+ p[0] = uint8(v >> 0)
184+ p[1] = uint8(v >> 8)
185+}
186+
187+func put4(p []byte, v uint32) {
188+ p[0] = uint8(v >> 0)
189+ p[1] = uint8(v >> 8)
190+ p[2] = uint8(v >> 16)
191+ p[3] = uint8(v >> 24)
192+}
193+
194+// writeBytes writes a length-prefixed byte slice to z.w.
195+func (z *Writer) writeBytes(b []byte) error {
196+ if len(b) > 0xffff {
197+ return errors.New("gzip.Write: Extra data is too large")
198+ }
199+ put2(z.buf[0:2], uint16(len(b)))
200+ _, err := z.w.Write(z.buf[0:2])
201+ if err != nil {
202+ return err
203+ }
204+ _, err = z.w.Write(b)
205+ return err
206+}
207+
208+// writeString writes a UTF-8 string s in GZIP's format to z.w.
209+// GZIP (RFC 1952) specifies that strings are NUL-terminated ISO 8859-1 (Latin-1).
210+func (z *Writer) writeString(s string) (err error) {
211+ // GZIP stores Latin-1 strings; error if non-Latin-1; convert if non-ASCII.
212+ needconv := false
213+ for _, v := range s {
214+ if v == 0 || v > 0xff {
215+ return errors.New("gzip.Write: non-Latin-1 header string")
216+ }
217+ if v > 0x7f {
218+ needconv = true
219+ }
220+ }
221+ if needconv {
222+ b := make([]byte, 0, len(s))
223+ for _, v := range s {
224+ b = append(b, byte(v))
225+ }
226+ _, err = z.w.Write(b)
227+ } else {
228+ _, err = io.WriteString(z.w, s)
229+ }
230+ if err != nil {
231+ return err
232+ }
233+ // GZIP strings are NUL-terminated.
234+ z.buf[0] = 0
235+ _, err = z.w.Write(z.buf[0:1])
236+ return err
237+}
238+
239+// compressCurrent will compress the data currently buffered
240+// This should only be called from the main writer/flush/closer
241+func (z *Writer) compressCurrent(flush bool) {
242+ r := result{}
243+ r.result = make(chan []byte, 1)
244+ r.notifyWritten = make(chan struct{}, 0)
245+ select {
246+ case z.results <- r:
247+ case <-z.pushedErr:
248+ return
249+ }
250+
251+ // If block given is more than twice the block size, split it.
252+ c := z.currentBuffer
253+ if len(c) > z.blockSize*2 {
254+ c = c[:z.blockSize]
255+ z.wg.Add(1)
256+ go z.compressBlock(c, z.prevTail, r, false)
257+ z.prevTail = c[len(c)-tailSize:]
258+ z.currentBuffer = z.currentBuffer[z.blockSize:]
259+ z.compressCurrent(flush)
260+ // Last one flushes if needed
261+ return
262+ }
263+
264+ z.wg.Add(1)
265+ go z.compressBlock(c, z.prevTail, r, z.closed)
266+ if len(c) > tailSize {
267+ z.prevTail = c[len(c)-tailSize:]
268+ } else {
269+ z.prevTail = nil
270+ }
271+ z.currentBuffer = z.dstPool.Get().([]byte)
272+ z.currentBuffer = z.currentBuffer[:0]
273+
274+ // Wait if flushing
275+ if flush {
276+ <-r.notifyWritten
277+ }
278+}
279+
280+// Returns an error if it has been set.
281+// Cannot be used by functions that are from internal goroutines.
282+func (z *Writer) checkError() error {
283+ z.errMu.RLock()
284+ err := z.err
285+ z.errMu.RUnlock()
286+ return err
287+}
288+
289+// Write writes a compressed form of p to the underlying io.Writer. The
290+// compressed bytes are not necessarily flushed to output until
291+// the Writer is closed or Flush() is called.
292+//
293+// The function will return quickly, if there are unused buffers.
294+// The sent slice (p) is copied, and the caller is free to re-use the buffer
295+// when the function returns.
296+//
297+// Errors that occur during compression will be reported later, and a nil error
298+// does not signify that the compression succeeded (since it is most likely still running)
299+// That means that the call that returns an error may not be the call that caused it.
300+// Only Flush and Close functions are guaranteed to return any errors up to that point.
301+func (z *Writer) Write(p []byte) (int, error) {
302+ if err := z.checkError(); err != nil {
303+ return 0, err
304+ }
305+ // Write the GZIP header lazily.
306+ if !z.wroteHeader {
307+ z.wroteHeader = true
308+ z.buf[0] = gzipID1
309+ z.buf[1] = gzipID2
310+ z.buf[2] = gzipDeflate
311+ z.buf[3] = 0
312+ if z.Extra != nil {
313+ z.buf[3] |= 0x04
314+ }
315+ if z.Name != "" {
316+ z.buf[3] |= 0x08
317+ }
318+ if z.Comment != "" {
319+ z.buf[3] |= 0x10
320+ }
321+ put4(z.buf[4:8], uint32(z.ModTime.Unix()))
322+ if z.level == BestCompression {
323+ z.buf[8] = 2
324+ } else if z.level == BestSpeed {
325+ z.buf[8] = 4
326+ } else {
327+ z.buf[8] = 0
328+ }
329+ z.buf[9] = z.OS
330+ var n int
331+ var err error
332+ n, err = z.w.Write(z.buf[0:10])
333+ if err != nil {
334+ z.pushError(err)
335+ return n, err
336+ }
337+ if z.Extra != nil {
338+ err = z.writeBytes(z.Extra)
339+ if err != nil {
340+ z.pushError(err)
341+ return n, err
342+ }
343+ }
344+ if z.Name != "" {
345+ err = z.writeString(z.Name)
346+ if err != nil {
347+ z.pushError(err)
348+ return n, err
349+ }
350+ }
351+ if z.Comment != "" {
352+ err = z.writeString(z.Comment)
353+ if err != nil {
354+ z.pushError(err)
355+ return n, err
356+ }
357+ }
358+ // Start receiving data from compressors
359+ go func() {
360+ listen := z.results
361+ for {
362+ r, ok := <-listen
363+ // If closed, we are finished.
364+ if !ok {
365+ return
366+ }
367+ buf := <-r.result
368+ n, err := z.w.Write(buf)
369+ if err != nil {
370+ z.pushError(err)
371+ close(r.notifyWritten)
372+ return
373+ }
374+ if n != len(buf) {
375+ z.pushError(fmt.Errorf("gzip: short write %d should be %d", n, len(buf)))
376+ close(r.notifyWritten)
377+ return
378+ }
379+ z.dstPool.Put(buf)
380+ close(r.notifyWritten)
381+ }
382+ }()
383+ z.currentBuffer = make([]byte, 0, z.blockSize)
384+ }
385+ q := p
386+ for len(q) > 0 {
387+ length := len(q)
388+ if length+len(z.currentBuffer) > z.blockSize {
389+ length = z.blockSize - len(z.currentBuffer)
390+ }
391+ z.digest.Write(q[:length])
392+ z.currentBuffer = append(z.currentBuffer, q[:length]...)
393+ if len(z.currentBuffer) >= z.blockSize {
394+ z.compressCurrent(false)
395+ if err := z.checkError(); err != nil {
396+ return len(p) - len(q) - length, err
397+ }
398+ }
399+ z.size += length
400+ q = q[length:]
401+ }
402+ return len(p), z.checkError()
403+}
404+
405+// Step 1: compresses buffer to buffer
406+// Step 2: send writer to channel
407+// Step 3: Close result channel to indicate we are done
408+func (z *Writer) compressBlock(p, prevTail []byte, r result, closed bool) {
409+ defer func() {
410+ close(r.result)
411+ z.wg.Done()
412+ }()
413+ buf := z.dstPool.Get().([]byte)
414+ dest := bytes.NewBuffer(buf[:0])
415+
416+ compressor := z.dictFlatePool.Get().(*flate.Writer)
417+ compressor.ResetDict(dest, prevTail)
418+ compressor.Write(p)
419+
420+ err := compressor.Flush()
421+ if err != nil {
422+ z.pushError(err)
423+ return
424+ }
425+ if closed {
426+ err = compressor.Close()
427+ if err != nil {
428+ z.pushError(err)
429+ return
430+ }
431+ }
432+ z.dictFlatePool.Put(compressor)
433+ // Read back buffer
434+ buf = dest.Bytes()
435+ r.result <- buf
436+}
437+
438+// Flush flushes any pending compressed data to the underlying writer.
439+//
440+// It is useful mainly in compressed network protocols, to ensure that
441+// a remote reader has enough data to reconstruct a packet. Flush does
442+// not return until the data has been written. If the underlying
443+// writer returns an error, Flush returns that error.
444+//
445+// In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH.
446+func (z *Writer) Flush() error {
447+ if err := z.checkError(); err != nil {
448+ return err
449+ }
450+ if z.closed {
451+ return nil
452+ }
453+ if !z.wroteHeader {
454+ _, err := z.Write(nil)
455+ if err != nil {
456+ return err
457+ }
458+ }
459+ // We send current block to compression
460+ z.compressCurrent(true)
461+
462+ return z.checkError()
463+}
464+
465+// UncompressedSize will return the number of bytes written.
466+// pgzip only, not a function in the official gzip package.
467+func (z *Writer) UncompressedSize() int {
468+ return z.size
469+}
470+
471+// Close closes the Writer, flushing any unwritten data to the underlying
472+// io.Writer, but does not close the underlying io.Writer.
473+func (z *Writer) Close() error {
474+ if err := z.checkError(); err != nil {
475+ return err
476+ }
477+ if z.closed {
478+ return nil
479+ }
480+
481+ z.closed = true
482+ if !z.wroteHeader {
483+ z.Write(nil)
484+ if err := z.checkError(); err != nil {
485+ return err
486+ }
487+ }
488+ z.compressCurrent(true)
489+ if err := z.checkError(); err != nil {
490+ return err
491+ }
492+ close(z.results)
493+ put4(z.buf[0:4], z.digest.Sum32())
494+ put4(z.buf[4:8], uint32(z.size))
495+ _, err := z.w.Write(z.buf[0:8])
496+ if err != nil {
497+ z.pushError(err)
498+ return err
499+ }
500+ return nil
501+}
vendor/github.com/xwb1989/sqlparser/CONTRIBUTORS.md+9−0
@@ -0,0 +1,9 @@
1+This project is originally a fork of [https://github.com/youtube/vitess](https://github.com/youtube/vitess)
2+Copyright Google Inc
3+
4+# Contributors
5+Wenbin Xiao 2015
6+Started this project and maintained it.
7+
8+Andrew Brampton 2017
9+Merged in multiple upstream fixes/changes.
vendor/github.com/xwb1989/sqlparser/LICENSE.md+201−0
@@ -0,0 +1,201 @@
1+ Apache License
2+ Version 2.0, January 2004
3+ http://www.apache.org/licenses/
4+
5+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6+
7+ 1. Definitions.
8+
9+ "License" shall mean the terms and conditions for use, reproduction,
10+ and distribution as defined by Sections 1 through 9 of this document.
11+
12+ "Licensor" shall mean the copyright owner or entity authorized by
13+ the copyright owner that is granting the License.
14+
15+ "Legal Entity" shall mean the union of the acting entity and all
16+ other entities that control, are controlled by, or are under common
17+ control with that entity. For the purposes of this definition,
18+ "control" means (i) the power, direct or indirect, to cause the
19+ direction or management of such entity, whether by contract or
20+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
21+ outstanding shares, or (iii) beneficial ownership of such entity.
22+
23+ "You" (or "Your") shall mean an individual or Legal Entity
24+ exercising permissions granted by this License.
25+
26+ "Source" form shall mean the preferred form for making modifications,
27+ including but not limited to software source code, documentation
28+ source, and configuration files.
29+
30+ "Object" form shall mean any form resulting from mechanical
31+ transformation or translation of a Source form, including but
32+ not limited to compiled object code, generated documentation,
33+ and conversions to other media types.
34+
35+ "Work" shall mean the work of authorship, whether in Source or
36+ Object form, made available under the License, as indicated by a
37+ copyright notice that is included in or attached to the work
38+ (an example is provided in the Appendix below).
39+
40+ "Derivative Works" shall mean any work, whether in Source or Object
41+ form, that is based on (or derived from) the Work and for which the
42+ editorial revisions, annotations, elaborations, or other modifications
43+ represent, as a whole, an original work of authorship. For the purposes
44+ of this License, Derivative Works shall not include works that remain
45+ separable from, or merely link (or bind by name) to the interfaces of,
46+ the Work and Derivative Works thereof.
47+
48+ "Contribution" shall mean any work of authorship, including
49+ the original version of the Work and any modifications or additions
50+ to that Work or Derivative Works thereof, that is intentionally
51+ submitted to Licensor for inclusion in the Work by the copyright owner
52+ or by an individual or Legal Entity authorized to submit on behalf of
53+ the copyright owner. For the purposes of this definition, "submitted"
54+ means any form of electronic, verbal, or written communication sent
55+ to the Licensor or its representatives, including but not limited to
56+ communication on electronic mailing lists, source code control systems,
57+ and issue tracking systems that are managed by, or on behalf of, the
58+ Licensor for the purpose of discussing and improving the Work, but
59+ excluding communication that is conspicuously marked or otherwise
60+ designated in writing by the copyright owner as "Not a Contribution."
61+
62+ "Contributor" shall mean Licensor and any individual or Legal Entity
63+ on behalf of whom a Contribution has been received by Licensor and
64+ subsequently incorporated within the Work.
65+
66+ 2. Grant of Copyright License. Subject to the terms and conditions of
67+ this License, each Contributor hereby grants to You a perpetual,
68+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69+ copyright license to reproduce, prepare Derivative Works of,
70+ publicly display, publicly perform, sublicense, and distribute the
71+ Work and such Derivative Works in Source or Object form.
72+
73+ 3. Grant of Patent License. Subject to the terms and conditions of
74+ this License, each Contributor hereby grants to You a perpetual,
75+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76+ (except as stated in this section) patent license to make, have made,
77+ use, offer to sell, sell, import, and otherwise transfer the Work,
78+ where such license applies only to those patent claims licensable
79+ by such Contributor that are necessarily infringed by their
80+ Contribution(s) alone or by combination of their Contribution(s)
81+ with the Work to which such Contribution(s) was submitted. If You
82+ institute patent litigation against any entity (including a
83+ cross-claim or counterclaim in a lawsuit) alleging that the Work
84+ or a Contribution incorporated within the Work constitutes direct
85+ or contributory patent infringement, then any patent licenses
86+ granted to You under this License for that Work shall terminate
87+ as of the date such litigation is filed.
88+
89+ 4. Redistribution. You may reproduce and distribute copies of the
90+ Work or Derivative Works thereof in any medium, with or without
91+ modifications, and in Source or Object form, provided that You
92+ meet the following conditions:
93+
94+ (a) You must give any other recipients of the Work or
95+ Derivative Works a copy of this License; and
96+
97+ (b) You must cause any modified files to carry prominent notices
98+ stating that You changed the files; and
99+
100+ (c) You must retain, in the Source form of any Derivative Works
101+ that You distribute, all copyright, patent, trademark, and
102+ attribution notices from the Source form of the Work,
103+ excluding those notices that do not pertain to any part of
104+ the Derivative Works; and
105+
106+ (d) If the Work includes a "NOTICE" text file as part of its
107+ distribution, then any Derivative Works that You distribute must
108+ include a readable copy of the attribution notices contained
109+ within such NOTICE file, excluding those notices that do not
110+ pertain to any part of the Derivative Works, in at least one
111+ of the following places: within a NOTICE text file distributed
112+ as part of the Derivative Works; within the Source form or
113+ documentation, if provided along with the Derivative Works; or,
114+ within a display generated by the Derivative Works, if and
115+ wherever such third-party notices normally appear. The contents
116+ of the NOTICE file are for informational purposes only and
117+ do not modify the License. You may add Your own attribution
118+ notices within Derivative Works that You distribute, alongside
119+ or as an addendum to the NOTICE text from the Work, provided
120+ that such additional attribution notices cannot be construed
121+ as modifying the License.
122+
123+ You may add Your own copyright statement to Your modifications and
124+ may provide additional or different license terms and conditions
125+ for use, reproduction, or distribution of Your modifications, or
126+ for any such Derivative Works as a whole, provided Your use,
127+ reproduction, and distribution of the Work otherwise complies with
128+ the conditions stated in this License.
129+
130+ 5. Submission of Contributions. Unless You explicitly state otherwise,
131+ any Contribution intentionally submitted for inclusion in the Work
132+ by You to the Licensor shall be under the terms and conditions of
133+ this License, without any additional terms or conditions.
134+ Notwithstanding the above, nothing herein shall supersede or modify
135+ the terms of any separate license agreement you may have executed
136+ with Licensor regarding such Contributions.
137+
138+ 6. Trademarks. This License does not grant permission to use the trade
139+ names, trademarks, service marks, or product names of the Licensor,
140+ except as required for reasonable and customary use in describing the
141+ origin of the Work and reproducing the content of the NOTICE file.
142+
143+ 7. Disclaimer of Warranty. Unless required by applicable law or
144+ agreed to in writing, Licensor provides the Work (and each
145+ Contributor provides its Contributions) on an "AS IS" BASIS,
146+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147+ implied, including, without limitation, any warranties or conditions
148+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149+ PARTICULAR PURPOSE. You are solely responsible for determining the
150+ appropriateness of using or redistributing the Work and assume any
151+ risks associated with Your exercise of permissions under this License.
152+
153+ 8. Limitation of Liability. In no event and under no legal theory,
154+ whether in tort (including negligence), contract, or otherwise,
155+ unless required by applicable law (such as deliberate and grossly
156+ negligent acts) or agreed to in writing, shall any Contributor be
157+ liable to You for damages, including any direct, indirect, special,
158+ incidental, or consequential damages of any character arising as a
159+ result of this License or out of the use or inability to use the
160+ Work (including but not limited to damages for loss of goodwill,
161+ work stoppage, computer failure or malfunction, or any and all
162+ other commercial damages or losses), even if such Contributor
163+ has been advised of the possibility of such damages.
164+
165+ 9. Accepting Warranty or Additional Liability. While redistributing
166+ the Work or Derivative Works thereof, You may choose to offer,
167+ and charge a fee for, acceptance of support, warranty, indemnity,
168+ or other liability obligations and/or rights consistent with this
169+ License. However, in accepting such obligations, You may act only
170+ on Your own behalf and on Your sole responsibility, not on behalf
171+ of any other Contributor, and only if You agree to indemnify,
172+ defend, and hold each Contributor harmless for any liability
173+ incurred by, or claims asserted against, such Contributor by reason
174+ of your accepting any such warranty or additional liability.
175+
176+ END OF TERMS AND CONDITIONS
177+
178+ APPENDIX: How to apply the Apache License to your work.
179+
180+ To apply the Apache License to your work, attach the following
181+ boilerplate notice, with the fields enclosed by brackets "[]"
182+ replaced with your own identifying information. (Don't include
183+ the brackets!) The text should be enclosed in the appropriate
184+ comment syntax for the file format. We also recommend that a
185+ file or class name and description of purpose be included on the
186+ same "printed page" as the copyright notice for easier
187+ identification within third-party archives.
188+
189+ Copyright [yyyy] [name of copyright owner]
190+
191+ Licensed under the Apache License, Version 2.0 (the "License");
192+ you may not use this file except in compliance with the License.
193+ You may obtain a copy of the License at
194+
195+ http://www.apache.org/licenses/LICENSE-2.0
196+
197+ Unless required by applicable law or agreed to in writing, software
198+ distributed under the License is distributed on an "AS IS" BASIS,
199+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200+ See the License for the specific language governing permissions and
201+ limitations under the License.
vendor/github.com/xwb1989/sqlparser/Makefile+22−0
@@ -0,0 +1,22 @@
1+# Copyright 2017 Google Inc.
2+#
3+# Licensed under the Apache License, Version 2.0 (the "License");
4+# you may not use this file except in compliance with the License.
5+# You may obtain a copy of the License at
6+#
7+# http://www.apache.org/licenses/LICENSE-2.0
8+#
9+# Unless required by applicable law or agreed to in writing, software
10+# distributed under the License is distributed on an "AS IS" BASIS,
11+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+# See the License for the specific language governing permissions and
13+# limitations under the License.
14+
15+MAKEFLAGS = -s
16+
17+sql.go: sql.y
18+ goyacc -o sql.go sql.y
19+ gofmt -w sql.go
20+
21+clean:
22+ rm -f y.output sql.go
vendor/github.com/xwb1989/sqlparser/README.md+150−0
@@ -0,0 +1,150 @@
1+# sqlparser [![Build Status](https://img.shields.io/travis/xwb1989/sqlparser.svg)](https://travis-ci.org/xwb1989/sqlparser) [![Coverage](https://img.shields.io/coveralls/xwb1989/sqlparser.svg)](https://coveralls.io/github/xwb1989/sqlparser) [![Report card](https://goreportcard.com/badge/github.com/xwb1989/sqlparser)](https://goreportcard.com/report/github.com/xwb1989/sqlparser) [![GoDoc](https://godoc.org/github.com/xwb1989/sqlparser?status.svg)](https://godoc.org/github.com/xwb1989/sqlparser)
2+
3+Go package for parsing MySQL SQL queries.
4+
5+## Notice
6+
7+The backbone of this repo is extracted from [vitessio/vitess](https://github.com/vitessio/vitess).
8+
9+Inside vitessio/vitess there is a very nicely written sql parser. However as it's not a self-contained application, I created this one.
10+It applies the same LICENSE as vitessio/vitess.
11+
12+## Usage
13+
14+```go
15+import (
16+ "github.com/xwb1989/sqlparser"
17+)
18+```
19+
20+Then use:
21+
22+```go
23+sql := "SELECT * FROM table WHERE a = 'abc'"
24+stmt, err := sqlparser.Parse(sql)
25+if err != nil {
26+ // Do something with the err
27+}
28+
29+// Otherwise do something with stmt
30+switch stmt := stmt.(type) {
31+case *sqlparser.Select:
32+ _ = stmt
33+case *sqlparser.Insert:
34+}
35+```
36+
37+Alternative to read many queries from a io.Reader:
38+
39+```go
40+r := strings.NewReader("INSERT INTO table1 VALUES (1, 'a'); INSERT INTO table2 VALUES (3, 4);")
41+
42+tokens := sqlparser.NewTokenizer(r)
43+for {
44+ stmt, err := sqlparser.ParseNext(tokens)
45+ if err == io.EOF {
46+ break
47+ }
48+ // Do something with stmt or err.
49+}
50+```
51+
52+See [parse_test.go](https://github.com/xwb1989/sqlparser/blob/master/parse_test.go) for more examples, or read the [godoc](https://godoc.org/github.com/xwb1989/sqlparser).
53+
54+
55+## Porting Instructions
56+
57+You only need the below if you plan to try and keep this library up to date with [vitessio/vitess](https://github.com/vitessio/vitess).
58+
59+### Keeping up to date
60+
61+```bash
62+shopt -s nullglob
63+VITESS=${GOPATH?}/src/vitess.io/vitess/go/
64+XWB1989=${GOPATH?}/src/github.com/xwb1989/sqlparser/
65+
66+# Create patches for everything that changed
67+LASTIMPORT=1b7879cb91f1dfe1a2dfa06fea96e951e3a7aec5
68+for path in ${VITESS?}/{vt/sqlparser,sqltypes,bytes2,hack}; do
69+ cd ${path}
70+ git format-patch ${LASTIMPORT?} .
71+done;
72+
73+# Apply patches to the dependencies
74+cd ${XWB1989?}
75+git am --directory dependency -p2 ${VITESS?}/{sqltypes,bytes2,hack}/*.patch
76+
77+# Apply the main patches to the repo
78+cd ${XWB1989?}
79+git am -p4 ${VITESS?}/vt/sqlparser/*.patch
80+
81+# If you encounter diff failures, manually fix them with
82+patch -p4 < .git/rebase-apply/patch
83+...
84+git add name_of_files
85+git am --continue
86+
87+# Cleanup
88+rm ${VITESS?}/{sqltypes,bytes2,hack}/*.patch ${VITESS?}/*.patch
89+
90+# and Finally update the LASTIMPORT in this README.
91+```
92+
93+### Fresh install
94+
95+TODO: Change these instructions to use git to copy the files, that'll make later patching easier.
96+
97+```bash
98+VITESS=${GOPATH?}/src/vitess.io/vitess/go/
99+XWB1989=${GOPATH?}/src/github.com/xwb1989/sqlparser/
100+
101+cd ${XWB1989?}
102+
103+# Copy all the code
104+cp -pr ${VITESS?}/vt/sqlparser/ .
105+cp -pr ${VITESS?}/sqltypes dependency
106+cp -pr ${VITESS?}/bytes2 dependency
107+cp -pr ${VITESS?}/hack dependency
108+
109+# Delete some code we haven't ported
110+rm dependency/sqltypes/arithmetic.go dependency/sqltypes/arithmetic_test.go dependency/sqltypes/event_token.go dependency/sqltypes/event_token_test.go dependency/sqltypes/proto3.go dependency/sqltypes/proto3_test.go dependency/sqltypes/query_response.go dependency/sqltypes/result.go dependency/sqltypes/result_test.go
111+
112+# Some automated fixes
113+
114+# Fix imports
115+sed -i '.bak' 's_vitess.io/vitess/go/vt/proto/query_github.com/xwb1989/sqlparser/dependency/querypb_g' *.go dependency/sqltypes/*.go
116+sed -i '.bak' 's_vitess.io/vitess/go/_github.com/xwb1989/sqlparser/dependency/_g' *.go dependency/sqltypes/*.go
117+
118+# Copy the proto, but basically drop everything we don't want
119+cp -pr ${VITESS?}/vt/proto/query dependency/querypb
120+
121+sed -i '.bak' 's_.*Descriptor.*__g' dependency/querypb/*.go
122+sed -i '.bak' 's_.*ProtoMessage.*__g' dependency/querypb/*.go
123+
124+sed -i '.bak' 's/proto.CompactTextString(m)/"TODO"/g' dependency/querypb/*.go
125+sed -i '.bak' 's/proto.EnumName/EnumName/g' dependency/querypb/*.go
126+
127+sed -i '.bak' 's/proto.Equal/reflect.DeepEqual/g' dependency/sqltypes/*.go
128+
129+# Remove the error library
130+sed -i '.bak' 's/vterrors.Errorf([^,]*, /fmt.Errorf(/g' *.go dependency/sqltypes/*.go
131+sed -i '.bak' 's/vterrors.New([^,]*, /errors.New(/g' *.go dependency/sqltypes/*.go
132+```
133+
134+### Testing
135+
136+```bash
137+VITESS=${GOPATH?}/src/vitess.io/vitess/go/
138+XWB1989=${GOPATH?}/src/github.com/xwb1989/sqlparser/
139+
140+cd ${XWB1989?}
141+
142+# Test, fix and repeat
143+go test ./...
144+
145+# Finally make some diffs (for later reference)
146+diff -u ${VITESS?}/sqltypes/ ${XWB1989?}/dependency/sqltypes/ > ${XWB1989?}/patches/sqltypes.patch
147+diff -u ${VITESS?}/bytes2/ ${XWB1989?}/dependency/bytes2/ > ${XWB1989?}/patches/bytes2.patch
148+diff -u ${VITESS?}/vt/proto/query/ ${XWB1989?}/dependency/querypb/ > ${XWB1989?}/patches/querypb.patch
149+diff -u ${VITESS?}/vt/sqlparser/ ${XWB1989?}/ > ${XWB1989?}/patches/sqlparser.patch
150+```
vendor/github.com/xwb1989/sqlparser/analyzer.go+343−0
@@ -0,0 +1,343 @@
1+/*
2+Copyright 2017 Google Inc.
3+
4+Licensed under the Apache License, Version 2.0 (the "License");
5+you may not use this file except in compliance with the License.
6+You may obtain a copy of the License at
7+
8+ http://www.apache.org/licenses/LICENSE-2.0
9+
10+Unless required by applicable law or agreed to in writing, software
11+distributed under the License is distributed on an "AS IS" BASIS,
12+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+See the License for the specific language governing permissions and
14+limitations under the License.
15+*/
16+
17+package sqlparser
18+
19+// analyzer.go contains utility analysis functions.
20+
21+import (
22+ "errors"
23+ "fmt"
24+ "strconv"
25+ "strings"
26+ "unicode"
27+
28+ "github.com/xwb1989/sqlparser/dependency/sqltypes"
29+)
30+
31+// These constants are used to identify the SQL statement type.
32+const (
33+ StmtSelect = iota
34+ StmtStream
35+ StmtInsert
36+ StmtReplace
37+ StmtUpdate
38+ StmtDelete
39+ StmtDDL
40+ StmtBegin
41+ StmtCommit
42+ StmtRollback
43+ StmtSet
44+ StmtShow
45+ StmtUse
46+ StmtOther
47+ StmtUnknown
48+ StmtComment
49+)
50+
51+// Preview analyzes the beginning of the query using a simpler and faster
52+// textual comparison to identify the statement type.
53+func Preview(sql string) int {
54+ trimmed := StripLeadingComments(sql)
55+
56+ firstWord := trimmed
57+ if end := strings.IndexFunc(trimmed, unicode.IsSpace); end != -1 {
58+ firstWord = trimmed[:end]
59+ }
60+ firstWord = strings.TrimLeftFunc(firstWord, func(r rune) bool { return !unicode.IsLetter(r) })
61+ // Comparison is done in order of priority.
62+ loweredFirstWord := strings.ToLower(firstWord)
63+ switch loweredFirstWord {
64+ case "select":
65+ return StmtSelect
66+ case "stream":
67+ return StmtStream
68+ case "insert":
69+ return StmtInsert
70+ case "replace":
71+ return StmtReplace
72+ case "update":
73+ return StmtUpdate
74+ case "delete":
75+ return StmtDelete
76+ }
77+ // For the following statements it is not sufficient to rely
78+ // on loweredFirstWord. This is because they are not statements
79+ // in the grammar and we are relying on Preview to parse them.
80+ // For instance, we don't want: "BEGIN JUNK" to be parsed
81+ // as StmtBegin.
82+ trimmedNoComments, _ := SplitMarginComments(trimmed)
83+ switch strings.ToLower(trimmedNoComments) {
84+ case "begin", "start transaction":
85+ return StmtBegin
86+ case "commit":
87+ return StmtCommit
88+ case "rollback":
89+ return StmtRollback
90+ }
91+ switch loweredFirstWord {
92+ case "create", "alter", "rename", "drop", "truncate":
93+ return StmtDDL
94+ case "set":
95+ return StmtSet
96+ case "show":
97+ return StmtShow
98+ case "use":
99+ return StmtUse
100+ case "analyze", "describe", "desc", "explain", "repair", "optimize":
101+ return StmtOther
102+ }
103+ if strings.Index(trimmed, "/*!") == 0 {
104+ return StmtComment
105+ }
106+ return StmtUnknown
107+}
108+
109+// StmtType returns the statement type as a string
110+func StmtType(stmtType int) string {
111+ switch stmtType {
112+ case StmtSelect:
113+ return "SELECT"
114+ case StmtStream:
115+ return "STREAM"
116+ case StmtInsert:
117+ return "INSERT"
118+ case StmtReplace:
119+ return "REPLACE"
120+ case StmtUpdate:
121+ return "UPDATE"
122+ case StmtDelete:
123+ return "DELETE"
124+ case StmtDDL:
125+ return "DDL"
126+ case StmtBegin:
127+ return "BEGIN"
128+ case StmtCommit:
129+ return "COMMIT"
130+ case StmtRollback:
131+ return "ROLLBACK"
132+ case StmtSet:
133+ return "SET"
134+ case StmtShow:
135+ return "SHOW"
136+ case StmtUse:
137+ return "USE"
138+ case StmtOther:
139+ return "OTHER"
140+ default:
141+ return "UNKNOWN"
142+ }
143+}
144+
145+// IsDML returns true if the query is an INSERT, UPDATE or DELETE statement.
146+func IsDML(sql string) bool {
147+ switch Preview(sql) {
148+ case StmtInsert, StmtReplace, StmtUpdate, StmtDelete:
149+ return true
150+ }
151+ return false
152+}
153+
154+// GetTableName returns the table name from the SimpleTableExpr
155+// only if it's a simple expression. Otherwise, it returns "".
156+func GetTableName(node SimpleTableExpr) TableIdent {
157+ if n, ok := node.(TableName); ok && n.Qualifier.IsEmpty() {
158+ return n.Name
159+ }
160+ // sub-select or '.' expression
161+ return NewTableIdent("")
162+}
163+
164+// IsColName returns true if the Expr is a *ColName.
165+func IsColName(node Expr) bool {
166+ _, ok := node.(*ColName)
167+ return ok
168+}
169+
170+// IsValue returns true if the Expr is a string, integral or value arg.
171+// NULL is not considered to be a value.
172+func IsValue(node Expr) bool {
173+ switch v := node.(type) {
174+ case *SQLVal:
175+ switch v.Type {
176+ case StrVal, HexVal, IntVal, ValArg:
177+ return true
178+ }
179+ }
180+ return false
181+}
182+
183+// IsNull returns true if the Expr is SQL NULL
184+func IsNull(node Expr) bool {
185+ switch node.(type) {
186+ case *NullVal:
187+ return true
188+ }
189+ return false
190+}
191+
192+// IsSimpleTuple returns true if the Expr is a ValTuple that
193+// contains simple values or if it's a list arg.
194+func IsSimpleTuple(node Expr) bool {
195+ switch vals := node.(type) {
196+ case ValTuple:
197+ for _, n := range vals {
198+ if !IsValue(n) {
199+ return false
200+ }
201+ }
202+ return true
203+ case ListArg:
204+ return true
205+ }
206+ // It's a subquery
207+ return false
208+}
209+
210+// NewPlanValue builds a sqltypes.PlanValue from an Expr.
211+func NewPlanValue(node Expr) (sqltypes.PlanValue, error) {
212+ switch node := node.(type) {
213+ case *SQLVal:
214+ switch node.Type {
215+ case ValArg:
216+ return sqltypes.PlanValue{Key: string(node.Val[1:])}, nil
217+ case IntVal:
218+ n, err := sqltypes.NewIntegral(string(node.Val))
219+ if err != nil {
220+ return sqltypes.PlanValue{}, fmt.Errorf("%v", err)
221+ }
222+ return sqltypes.PlanValue{Value: n}, nil
223+ case StrVal:
224+ return sqltypes.PlanValue{Value: sqltypes.MakeTrusted(sqltypes.VarBinary, node.Val)}, nil
225+ case HexVal:
226+ v, err := node.HexDecode()
227+ if err != nil {
228+ return sqltypes.PlanValue{}, fmt.Errorf("%v", err)
229+ }
230+ return sqltypes.PlanValue{Value: sqltypes.MakeTrusted(sqltypes.VarBinary, v)}, nil
231+ }
232+ case ListArg:
233+ return sqltypes.PlanValue{ListKey: string(node[2:])}, nil
234+ case ValTuple:
235+ pv := sqltypes.PlanValue{
236+ Values: make([]sqltypes.PlanValue, 0, len(node)),
237+ }
238+ for _, val := range node {
239+ innerpv, err := NewPlanValue(val)
240+ if err != nil {
241+ return sqltypes.PlanValue{}, err
242+ }
243+ if innerpv.ListKey != "" || innerpv.Values != nil {
244+ return sqltypes.PlanValue{}, errors.New("unsupported: nested lists")
245+ }
246+ pv.Values = append(pv.Values, innerpv)
247+ }
248+ return pv, nil
249+ case *NullVal:
250+ return sqltypes.PlanValue{}, nil
251+ }
252+ return sqltypes.PlanValue{}, fmt.Errorf("expression is too complex '%v'", String(node))
253+}
254+
255+// StringIn is a convenience function that returns
256+// true if str matches any of the values.
257+func StringIn(str string, values ...string) bool {
258+ for _, val := range values {
259+ if str == val {
260+ return true
261+ }
262+ }
263+ return false
264+}
265+
266+// SetKey is the extracted key from one SetExpr
267+type SetKey struct {
268+ Key string
269+ Scope string
270+}
271+
272+// ExtractSetValues returns a map of key-value pairs
273+// if the query is a SET statement. Values can be bool, int64 or string.
274+// Since set variable names are case insensitive, all keys are returned
275+// as lower case.
276+func ExtractSetValues(sql string) (keyValues map[SetKey]interface{}, scope string, err error) {
277+ stmt, err := Parse(sql)
278+ if err != nil {
279+ return nil, "", err
280+ }
281+ setStmt, ok := stmt.(*Set)
282+ if !ok {
283+ return nil, "", fmt.Errorf("ast did not yield *sqlparser.Set: %T", stmt)
284+ }
285+ result := make(map[SetKey]interface{})
286+ for _, expr := range setStmt.Exprs {
287+ scope := SessionStr
288+ key := expr.Name.Lowered()
289+ switch {
290+ case strings.HasPrefix(key, "@@global."):
291+ scope = GlobalStr
292+ key = strings.TrimPrefix(key, "@@global.")
293+ case strings.HasPrefix(key, "@@session."):
294+ key = strings.TrimPrefix(key, "@@session.")
295+ case strings.HasPrefix(key, "@@"):
296+ key = strings.TrimPrefix(key, "@@")
297+ }
298+
299+ if strings.HasPrefix(expr.Name.Lowered(), "@@") {
300+ if setStmt.Scope != "" && scope != "" {
301+ return nil, "", fmt.Errorf("unsupported in set: mixed using of variable scope")
302+ }
303+ _, out := NewStringTokenizer(key).Scan()
304+ key = string(out)
305+ }
306+
307+ setKey := SetKey{
308+ Key: key,
309+ Scope: scope,
310+ }
311+
312+ switch expr := expr.Expr.(type) {
313+ case *SQLVal:
314+ switch expr.Type {
315+ case StrVal:
316+ result[setKey] = strings.ToLower(string(expr.Val))
317+ case IntVal:
318+ num, err := strconv.ParseInt(string(expr.Val), 0, 64)
319+ if err != nil {
320+ return nil, "", err
321+ }
322+ result[setKey] = num
323+ default:
324+ return nil, "", fmt.Errorf("invalid value type: %v", String(expr))
325+ }
326+ case BoolVal:
327+ var val int64
328+ if expr {
329+ val = 1
330+ }
331+ result[setKey] = val
332+ case *ColName:
333+ result[setKey] = expr.Name.String()
334+ case *NullVal:
335+ result[setKey] = nil
336+ case *Default:
337+ result[setKey] = "default"
338+ default:
339+ return nil, "", fmt.Errorf("invalid syntax: %s", String(expr))
340+ }
341+ }
342+ return result, strings.ToLower(setStmt.Scope), nil
343+}
vendor/github.com/xwb1989/sqlparser/ast.go+3450−0
@@ -0,0 +1,3450 @@
1+/*
2+Copyright 2017 Google Inc.
3+
4+Licensed under the Apache License, Version 2.0 (the "License");
5+you may not use this file except in compliance with the License.
6+You may obtain a copy of the License at
7+
8+ http://www.apache.org/licenses/LICENSE-2.0
9+
10+Unless required by applicable law or agreed to in writing, software
11+distributed under the License is distributed on an "AS IS" BASIS,
12+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+See the License for the specific language governing permissions and
14+limitations under the License.
15+*/
16+
17+package sqlparser
18+
19+import (
20+ "bytes"
21+ "encoding/hex"
22+ "encoding/json"
23+ "fmt"
24+ "io"
25+ "log"
26+ "strings"
27+
28+ "github.com/xwb1989/sqlparser/dependency/querypb"
29+ "github.com/xwb1989/sqlparser/dependency/sqltypes"
30+)
31+
32+// Instructions for creating new types: If a type
33+// needs to satisfy an interface, declare that function
34+// along with that interface. This will help users
35+// identify the list of types to which they can assert
36+// those interfaces.
37+// If the member of a type has a string with a predefined
38+// list of values, declare those values as const following
39+// the type.
40+// For interfaces that define dummy functions to consolidate
41+// a set of types, define the function as iTypeName.
42+// This will help avoid name collisions.
43+
44+// Parse parses the SQL in full and returns a Statement, which
45+// is the AST representation of the query. If a DDL statement
46+// is partially parsed but still contains a syntax error, the
47+// error is ignored and the DDL is returned anyway.
48+func Parse(sql string) (Statement, error) {
49+ tokenizer := NewStringTokenizer(sql)
50+ if yyParse(tokenizer) != 0 {
51+ if tokenizer.partialDDL != nil {
52+ log.Printf("ignoring error parsing DDL '%s': %v", sql, tokenizer.LastError)
53+ tokenizer.ParseTree = tokenizer.partialDDL
54+ return tokenizer.ParseTree, nil
55+ }
56+ return nil, tokenizer.LastError
57+ }
58+ return tokenizer.ParseTree, nil
59+}
60+
61+// ParseStrictDDL is the same as Parse except it errors on
62+// partially parsed DDL statements.
63+func ParseStrictDDL(sql string) (Statement, error) {
64+ tokenizer := NewStringTokenizer(sql)
65+ if yyParse(tokenizer) != 0 {
66+ return nil, tokenizer.LastError
67+ }
68+ return tokenizer.ParseTree, nil
69+}
70+
71+// ParseNext parses a single SQL statement from the tokenizer
72+// returning a Statement which is the AST representation of the query.
73+// The tokenizer will always read up to the end of the statement, allowing for
74+// the next call to ParseNext to parse any subsequent SQL statements. When
75+// there are no more statements to parse, a error of io.EOF is returned.
76+func ParseNext(tokenizer *Tokenizer) (Statement, error) {
77+ if tokenizer.lastChar == ';' {
78+ tokenizer.next()
79+ tokenizer.skipBlank()
80+ }
81+ if tokenizer.lastChar == eofChar {
82+ return nil, io.EOF
83+ }
84+
85+ tokenizer.reset()
86+ tokenizer.multi = true
87+ if yyParse(tokenizer) != 0 {
88+ if tokenizer.partialDDL != nil {
89+ tokenizer.ParseTree = tokenizer.partialDDL
90+ return tokenizer.ParseTree, nil
91+ }
92+ return nil, tokenizer.LastError
93+ }
94+ return tokenizer.ParseTree, nil
95+}
96+
97+// SplitStatement returns the first sql statement up to either a ; or EOF
98+// and the remainder from the given buffer
99+func SplitStatement(blob string) (string, string, error) {
100+ tokenizer := NewStringTokenizer(blob)
101+ tkn := 0
102+ for {
103+ tkn, _ = tokenizer.Scan()
104+ if tkn == 0 || tkn == ';' || tkn == eofChar {
105+ break
106+ }
107+ }
108+ if tokenizer.LastError != nil {
109+ return "", "", tokenizer.LastError
110+ }
111+ if tkn == ';' {
112+ return blob[:tokenizer.Position-2], blob[tokenizer.Position-1:], nil
113+ }
114+ return blob, "", nil
115+}
116+
117+// SplitStatementToPieces split raw sql statement that may have multi sql pieces to sql pieces
118+// returns the sql pieces blob contains; or error if sql cannot be parsed
119+func SplitStatementToPieces(blob string) (pieces []string, err error) {
120+ pieces = make([]string, 0, 16)
121+ tokenizer := NewStringTokenizer(blob)
122+
123+ tkn := 0
124+ var stmt string
125+ stmtBegin := 0
126+ for {
127+ tkn, _ = tokenizer.Scan()
128+ if tkn == ';' {
129+ stmt = blob[stmtBegin : tokenizer.Position-2]
130+ pieces = append(pieces, stmt)
131+ stmtBegin = tokenizer.Position - 1
132+
133+ } else if tkn == 0 || tkn == eofChar {
134+ blobTail := tokenizer.Position - 2
135+
136+ if stmtBegin < blobTail {
137+ stmt = blob[stmtBegin : blobTail+1]
138+ pieces = append(pieces, stmt)
139+ }
140+ break
141+ }
142+ }
143+
144+ err = tokenizer.LastError
145+ return
146+}
147+
148+// SQLNode defines the interface for all nodes
149+// generated by the parser.
150+type SQLNode interface {
151+ Format(buf *TrackedBuffer)
152+ // walkSubtree calls visit on all underlying nodes
153+ // of the subtree, but not the current one. Walking
154+ // must be interrupted if visit returns an error.
155+ walkSubtree(visit Visit) error
156+}
157+
158+// Visit defines the signature of a function that
159+// can be used to visit all nodes of a parse tree.
160+type Visit func(node SQLNode) (kontinue bool, err error)
161+
162+// Walk calls visit on every node.
163+// If visit returns true, the underlying nodes
164+// are also visited. If it returns an error, walking
165+// is interrupted, and the error is returned.
166+func Walk(visit Visit, nodes ...SQLNode) error {
167+ for _, node := range nodes {
168+ if node == nil {
169+ continue
170+ }
171+ kontinue, err := visit(node)
172+ if err != nil {
173+ return err
174+ }
175+ if kontinue {
176+ err = node.walkSubtree(visit)
177+ if err != nil {
178+ return err
179+ }
180+ }
181+ }
182+ return nil
183+}
184+
185+// String returns a string representation of an SQLNode.
186+func String(node SQLNode) string {
187+ if node == nil {
188+ return "<nil>"
189+ }
190+
191+ buf := NewTrackedBuffer(nil)
192+ buf.Myprintf("%v", node)
193+ return buf.String()
194+}
195+
196+// Append appends the SQLNode to the buffer.
197+func Append(buf *bytes.Buffer, node SQLNode) {
198+ tbuf := &TrackedBuffer{
199+ Buffer: buf,
200+ }
201+ node.Format(tbuf)
202+}
203+
204+// Statement represents a statement.
205+type Statement interface {
206+ iStatement()
207+ SQLNode
208+}
209+
210+func (*Union) iStatement() {}
211+func (*Select) iStatement() {}
212+func (*Stream) iStatement() {}
213+func (*Insert) iStatement() {}
214+func (*Update) iStatement() {}
215+func (*Delete) iStatement() {}
216+func (*Set) iStatement() {}
217+func (*DBDDL) iStatement() {}
218+func (*DDL) iStatement() {}
219+func (*Show) iStatement() {}
220+func (*Use) iStatement() {}
221+func (*Begin) iStatement() {}
222+func (*Commit) iStatement() {}
223+func (*Rollback) iStatement() {}
224+func (*OtherRead) iStatement() {}
225+func (*OtherAdmin) iStatement() {}
226+
227+// ParenSelect can actually not be a top level statement,
228+// but we have to allow it because it's a requirement
229+// of SelectStatement.
230+func (*ParenSelect) iStatement() {}
231+
232+// SelectStatement any SELECT statement.
233+type SelectStatement interface {
234+ iSelectStatement()
235+ iStatement()
236+ iInsertRows()
237+ AddOrder(*Order)
238+ SetLimit(*Limit)
239+ SQLNode
240+}
241+
242+func (*Select) iSelectStatement() {}
243+func (*Union) iSelectStatement() {}
244+func (*ParenSelect) iSelectStatement() {}
245+
246+// Select represents a SELECT statement.
247+type Select struct {
248+ Cache string
249+ Comments Comments
250+ Distinct string
251+ Hints string
252+ SelectExprs SelectExprs
253+ From TableExprs
254+ Where *Where
255+ GroupBy GroupBy
256+ Having *Where
257+ OrderBy OrderBy
258+ Limit *Limit
259+ Lock string
260+}
261+
262+// Select.Distinct
263+const (
264+ DistinctStr = "distinct "
265+ StraightJoinHint = "straight_join "
266+)
267+
268+// Select.Lock
269+const (
270+ ForUpdateStr = " for update"
271+ ShareModeStr = " lock in share mode"
272+)
273+
274+// Select.Cache
275+const (
276+ SQLCacheStr = "sql_cache "
277+ SQLNoCacheStr = "sql_no_cache "
278+)
279+
280+// AddOrder adds an order by element
281+func (node *Select) AddOrder(order *Order) {
282+ node.OrderBy = append(node.OrderBy, order)
283+}
284+
285+// SetLimit sets the limit clause
286+func (node *Select) SetLimit(limit *Limit) {
287+ node.Limit = limit
288+}
289+
290+// Format formats the node.
291+func (node *Select) Format(buf *TrackedBuffer) {
292+ buf.Myprintf("select %v%s%s%s%v from %v%v%v%v%v%v%s",
293+ node.Comments, node.Cache, node.Distinct, node.Hints, node.SelectExprs,
294+ node.From, node.Where,
295+ node.GroupBy, node.Having, node.OrderBy,
296+ node.Limit, node.Lock)
297+}
298+
299+func (node *Select) walkSubtree(visit Visit) error {
300+ if node == nil {
301+ return nil
302+ }
303+ return Walk(
304+ visit,
305+ node.Comments,
306+ node.SelectExprs,
307+ node.From,
308+ node.Where,
309+ node.GroupBy,
310+ node.Having,
311+ node.OrderBy,
312+ node.Limit,
313+ )
314+}
315+
316+// AddWhere adds the boolean expression to the
317+// WHERE clause as an AND condition. If the expression
318+// is an OR clause, it parenthesizes it. Currently,
319+// the OR operator is the only one that's lower precedence
320+// than AND.
321+func (node *Select) AddWhere(expr Expr) {
322+ if _, ok := expr.(*OrExpr); ok {
323+ expr = &ParenExpr{Expr: expr}
324+ }
325+ if node.Where == nil {
326+ node.Where = &Where{
327+ Type: WhereStr,
328+ Expr: expr,
329+ }
330+ return
331+ }
332+ node.Where.Expr = &AndExpr{
333+ Left: node.Where.Expr,
334+ Right: expr,
335+ }
336+ return
337+}
338+
339+// AddHaving adds the boolean expression to the
340+// HAVING clause as an AND condition. If the expression
341+// is an OR clause, it parenthesizes it. Currently,
342+// the OR operator is the only one that's lower precedence
343+// than AND.
344+func (node *Select) AddHaving(expr Expr) {
345+ if _, ok := expr.(*OrExpr); ok {
346+ expr = &ParenExpr{Expr: expr}
347+ }
348+ if node.Having == nil {
349+ node.Having = &Where{
350+ Type: HavingStr,
351+ Expr: expr,
352+ }
353+ return
354+ }
355+ node.Having.Expr = &AndExpr{
356+ Left: node.Having.Expr,
357+ Right: expr,
358+ }
359+ return
360+}
361+
362+// ParenSelect is a parenthesized SELECT statement.
363+type ParenSelect struct {
364+ Select SelectStatement
365+}
366+
367+// AddOrder adds an order by element
368+func (node *ParenSelect) AddOrder(order *Order) {
369+ panic("unreachable")
370+}
371+
372+// SetLimit sets the limit clause
373+func (node *ParenSelect) SetLimit(limit *Limit) {
374+ panic("unreachable")
375+}
376+
377+// Format formats the node.
378+func (node *ParenSelect) Format(buf *TrackedBuffer) {
379+ buf.Myprintf("(%v)", node.Select)
380+}
381+
382+func (node *ParenSelect) walkSubtree(visit Visit) error {
383+ if node == nil {
384+ return nil
385+ }
386+ return Walk(
387+ visit,
388+ node.Select,
389+ )
390+}
391+
392+// Union represents a UNION statement.
393+type Union struct {
394+ Type string
395+ Left, Right SelectStatement
396+ OrderBy OrderBy
397+ Limit *Limit
398+ Lock string
399+}
400+
401+// Union.Type
402+const (
403+ UnionStr = "union"
404+ UnionAllStr = "union all"
405+ UnionDistinctStr = "union distinct"
406+)
407+
408+// AddOrder adds an order by element
409+func (node *Union) AddOrder(order *Order) {
410+ node.OrderBy = append(node.OrderBy, order)
411+}
412+
413+// SetLimit sets the limit clause
414+func (node *Union) SetLimit(limit *Limit) {
415+ node.Limit = limit
416+}
417+
418+// Format formats the node.
419+func (node *Union) Format(buf *TrackedBuffer) {
420+ buf.Myprintf("%v %s %v%v%v%s", node.Left, node.Type, node.Right,
421+ node.OrderBy, node.Limit, node.Lock)
422+}
423+
424+func (node *Union) walkSubtree(visit Visit) error {
425+ if node == nil {
426+ return nil
427+ }
428+ return Walk(
429+ visit,
430+ node.Left,
431+ node.Right,
432+ )
433+}
434+
435+// Stream represents a SELECT statement.
436+type Stream struct {
437+ Comments Comments
438+ SelectExpr SelectExpr
439+ Table TableName
440+}
441+
442+// Format formats the node.
443+func (node *Stream) Format(buf *TrackedBuffer) {
444+ buf.Myprintf("stream %v%v from %v",
445+ node.Comments, node.SelectExpr, node.Table)
446+}
447+
448+func (node *Stream) walkSubtree(visit Visit) error {
449+ if node == nil {
450+ return nil
451+ }
452+ return Walk(
453+ visit,
454+ node.Comments,
455+ node.SelectExpr,
456+ node.Table,
457+ )
458+}
459+
460+// Insert represents an INSERT or REPLACE statement.
461+// Per the MySQL docs, http://dev.mysql.com/doc/refman/5.7/en/replace.html
462+// Replace is the counterpart to `INSERT IGNORE`, and works exactly like a
463+// normal INSERT except if the row exists. In that case it first deletes
464+// the row and re-inserts with new values. For that reason we keep it as an Insert struct.
465+// Replaces are currently disallowed in sharded schemas because
466+// of the implications the deletion part may have on vindexes.
467+// If you add fields here, consider adding them to calls to validateSubquerySamePlan.
468+type Insert struct {
469+ Action string
470+ Comments Comments
471+ Ignore string
472+ Table TableName
473+ Partitions Partitions
474+ Columns Columns
475+ Rows InsertRows
476+ OnDup OnDup
477+}
478+
479+// DDL strings.
480+const (
481+ InsertStr = "insert"
482+ ReplaceStr = "replace"
483+)
484+
485+// Format formats the node.
486+func (node *Insert) Format(buf *TrackedBuffer) {
487+ buf.Myprintf("%s %v%sinto %v%v%v %v%v",
488+ node.Action,
489+ node.Comments, node.Ignore,
490+ node.Table, node.Partitions, node.Columns, node.Rows, node.OnDup)
491+}
492+
493+func (node *Insert) walkSubtree(visit Visit) error {
494+ if node == nil {
495+ return nil
496+ }
497+ return Walk(
498+ visit,
499+ node.Comments,
500+ node.Table,
501+ node.Columns,
502+ node.Rows,
503+ node.OnDup,
504+ )
505+}
506+
507+// InsertRows represents the rows for an INSERT statement.
508+type InsertRows interface {
509+ iInsertRows()
510+ SQLNode
511+}
512+
513+func (*Select) iInsertRows() {}
514+func (*Union) iInsertRows() {}
515+func (Values) iInsertRows() {}
516+func (*ParenSelect) iInsertRows() {}
517+
518+// Update represents an UPDATE statement.
519+// If you add fields here, consider adding them to calls to validateSubquerySamePlan.
520+type Update struct {
521+ Comments Comments
522+ TableExprs TableExprs
523+ Exprs UpdateExprs
524+ Where *Where
525+ OrderBy OrderBy
526+ Limit *Limit
527+}
528+
529+// Format formats the node.
530+func (node *Update) Format(buf *TrackedBuffer) {
531+ buf.Myprintf("update %v%v set %v%v%v%v",
532+ node.Comments, node.TableExprs,
533+ node.Exprs, node.Where, node.OrderBy, node.Limit)
534+}
535+
536+func (node *Update) walkSubtree(visit Visit) error {
537+ if node == nil {
538+ return nil
539+ }
540+ return Walk(
541+ visit,
542+ node.Comments,
543+ node.TableExprs,
544+ node.Exprs,
545+ node.Where,
546+ node.OrderBy,
547+ node.Limit,
548+ )
549+}
550+
551+// Delete represents a DELETE statement.
552+// If you add fields here, consider adding them to calls to validateSubquerySamePlan.
553+type Delete struct {
554+ Comments Comments
555+ Targets TableNames
556+ TableExprs TableExprs
557+ Partitions Partitions
558+ Where *Where
559+ OrderBy OrderBy
560+ Limit *Limit
561+}
562+
563+// Format formats the node.
564+func (node *Delete) Format(buf *TrackedBuffer) {
565+ buf.Myprintf("delete %v", node.Comments)
566+ if node.Targets != nil {
567+ buf.Myprintf("%v ", node.Targets)
568+ }
569+ buf.Myprintf("from %v%v%v%v%v", node.TableExprs, node.Partitions, node.Where, node.OrderBy, node.Limit)
570+}
571+
572+func (node *Delete) walkSubtree(visit Visit) error {
573+ if node == nil {
574+ return nil
575+ }
576+ return Walk(
577+ visit,
578+ node.Comments,
579+ node.Targets,
580+ node.TableExprs,
581+ node.Where,
582+ node.OrderBy,
583+ node.Limit,
584+ )
585+}
586+
587+// Set represents a SET statement.
588+type Set struct {
589+ Comments Comments
590+ Exprs SetExprs
591+ Scope string
592+}
593+
594+// Set.Scope or Show.Scope
595+const (
596+ SessionStr = "session"
597+ GlobalStr = "global"
598+)
599+
600+// Format formats the node.
601+func (node *Set) Format(buf *TrackedBuffer) {
602+ if node.Scope == "" {
603+ buf.Myprintf("set %v%v", node.Comments, node.Exprs)
604+ } else {
605+ buf.Myprintf("set %v%s %v", node.Comments, node.Scope, node.Exprs)
606+ }
607+}
608+
609+func (node *Set) walkSubtree(visit Visit) error {
610+ if node == nil {
611+ return nil
612+ }
613+ return Walk(
614+ visit,
615+ node.Comments,
616+ node.Exprs,
617+ )
618+}
619+
620+// DBDDL represents a CREATE, DROP database statement.
621+type DBDDL struct {
622+ Action string
623+ DBName string
624+ IfExists bool
625+ Collate string
626+ Charset string
627+}
628+
629+// Format formats the node.
630+func (node *DBDDL) Format(buf *TrackedBuffer) {
631+ switch node.Action {
632+ case CreateStr:
633+ buf.WriteString(fmt.Sprintf("%s database %s", node.Action, node.DBName))
634+ case DropStr:
635+ exists := ""
636+ if node.IfExists {
637+ exists = " if exists"
638+ }
639+ buf.WriteString(fmt.Sprintf("%s database%s %v", node.Action, exists, node.DBName))
640+ }
641+}
642+
643+// walkSubtree walks the nodes of the subtree.
644+func (node *DBDDL) walkSubtree(visit Visit) error {
645+ return nil
646+}
647+
648+// DDL represents a CREATE, ALTER, DROP, RENAME or TRUNCATE statement.
649+// Table is set for AlterStr, DropStr, RenameStr, TruncateStr
650+// NewName is set for AlterStr, CreateStr, RenameStr.
651+// VindexSpec is set for CreateVindexStr, DropVindexStr, AddColVindexStr, DropColVindexStr
652+// VindexCols is set for AddColVindexStr
653+type DDL struct {
654+ Action string
655+ Table TableName
656+ NewName TableName
657+ IfExists bool
658+ TableSpec *TableSpec
659+ PartitionSpec *PartitionSpec
660+ VindexSpec *VindexSpec
661+ VindexCols []ColIdent
662+}
663+
664+// DDL strings.
665+const (
666+ CreateStr = "create"
667+ AlterStr = "alter"
668+ DropStr = "drop"
669+ RenameStr = "rename"
670+ TruncateStr = "truncate"
671+ CreateVindexStr = "create vindex"
672+ AddColVindexStr = "add vindex"
673+ DropColVindexStr = "drop vindex"
674+
675+ // Vindex DDL param to specify the owner of a vindex
676+ VindexOwnerStr = "owner"
677+)
678+
679+// Format formats the node.
680+func (node *DDL) Format(buf *TrackedBuffer) {
681+ switch node.Action {
682+ case CreateStr:
683+ if node.TableSpec == nil {
684+ buf.Myprintf("%s table %v", node.Action, node.NewName)
685+ } else {
686+ buf.Myprintf("%s table %v %v", node.Action, node.NewName, node.TableSpec)
687+ }
688+ case DropStr:
689+ exists := ""
690+ if node.IfExists {
691+ exists = " if exists"
692+ }
693+ buf.Myprintf("%s table%s %v", node.Action, exists, node.Table)
694+ case RenameStr:
695+ buf.Myprintf("%s table %v to %v", node.Action, node.Table, node.NewName)
696+ case AlterStr:
697+ if node.PartitionSpec != nil {
698+ buf.Myprintf("%s table %v %v", node.Action, node.Table, node.PartitionSpec)
699+ } else {
700+ buf.Myprintf("%s table %v", node.Action, node.Table)
701+ }
702+ case CreateVindexStr:
703+ buf.Myprintf("%s %v %v", node.Action, node.VindexSpec.Name, node.VindexSpec)
704+ case AddColVindexStr:
705+ buf.Myprintf("alter table %v %s %v (", node.Table, node.Action, node.VindexSpec.Name)
706+ for i, col := range node.VindexCols {
707+ if i != 0 {
708+ buf.Myprintf(", %v", col)
709+ } else {
710+ buf.Myprintf("%v", col)
711+ }
712+ }
713+ buf.Myprintf(")")
714+ if node.VindexSpec.Type.String() != "" {
715+ buf.Myprintf(" %v", node.VindexSpec)
716+ }
717+ case DropColVindexStr:
718+ buf.Myprintf("alter table %v %s %v", node.Table, node.Action, node.VindexSpec.Name)
719+ default:
720+ buf.Myprintf("%s table %v", node.Action, node.Table)
721+ }
722+}
723+
724+func (node *DDL) walkSubtree(visit Visit) error {
725+ if node == nil {
726+ return nil
727+ }
728+ return Walk(
729+ visit,
730+ node.Table,
731+ node.NewName,
732+ )
733+}
734+
735+// Partition strings
736+const (
737+ ReorganizeStr = "reorganize partition"
738+)
739+
740+// PartitionSpec describe partition actions (for alter and create)
741+type PartitionSpec struct {
742+ Action string
743+ Name ColIdent
744+ Definitions []*PartitionDefinition
745+}
746+
747+// Format formats the node.
748+func (node *PartitionSpec) Format(buf *TrackedBuffer) {
749+ switch node.Action {
750+ case ReorganizeStr:
751+ buf.Myprintf("%s %v into (", node.Action, node.Name)
752+ var prefix string
753+ for _, pd := range node.Definitions {
754+ buf.Myprintf("%s%v", prefix, pd)
755+ prefix = ", "
756+ }
757+ buf.Myprintf(")")
758+ default:
759+ panic("unimplemented")
760+ }
761+}
762+
763+func (node *PartitionSpec) walkSubtree(visit Visit) error {
764+ if node == nil {
765+ return nil
766+ }
767+ if err := Walk(visit, node.Name); err != nil {
768+ return err
769+ }
770+ for _, def := range node.Definitions {
771+ if err := Walk(visit, def); err != nil {
772+ return err
773+ }
774+ }
775+ return nil
776+}
777+
778+// PartitionDefinition describes a very minimal partition definition
779+type PartitionDefinition struct {
780+ Name ColIdent
781+ Limit Expr
782+ Maxvalue bool
783+}
784+
785+// Format formats the node
786+func (node *PartitionDefinition) Format(buf *TrackedBuffer) {
787+ if !node.Maxvalue {
788+ buf.Myprintf("partition %v values less than (%v)", node.Name, node.Limit)
789+ } else {
790+ buf.Myprintf("partition %v values less than (maxvalue)", node.Name)
791+ }
792+}
793+
794+func (node *PartitionDefinition) walkSubtree(visit Visit) error {
795+ if node == nil {
796+ return nil
797+ }
798+ return Walk(
799+ visit,
800+ node.Name,
801+ node.Limit,
802+ )
803+}
804+
805+// TableSpec describes the structure of a table from a CREATE TABLE statement
806+type TableSpec struct {
807+ Columns []*ColumnDefinition
808+ Indexes []*IndexDefinition
809+ Options string
810+}
811+
812+// Format formats the node.
813+func (ts *TableSpec) Format(buf *TrackedBuffer) {
814+ buf.Myprintf("(\n")
815+ for i, col := range ts.Columns {
816+ if i == 0 {
817+ buf.Myprintf("\t%v", col)
818+ } else {
819+ buf.Myprintf(",\n\t%v", col)
820+ }
821+ }
822+ for _, idx := range ts.Indexes {
823+ buf.Myprintf(",\n\t%v", idx)
824+ }
825+
826+ buf.Myprintf("\n)%s", strings.Replace(ts.Options, ", ", ",\n ", -1))
827+}
828+
829+// AddColumn appends the given column to the list in the spec
830+func (ts *TableSpec) AddColumn(cd *ColumnDefinition) {
831+ ts.Columns = append(ts.Columns, cd)
832+}
833+
834+// AddIndex appends the given index to the list in the spec
835+func (ts *TableSpec) AddIndex(id *IndexDefinition) {
836+ ts.Indexes = append(ts.Indexes, id)
837+}
838+
839+func (ts *TableSpec) walkSubtree(visit Visit) error {
840+ if ts == nil {
841+ return nil
842+ }
843+
844+ for _, n := range ts.Columns {
845+ if err := Walk(visit, n); err != nil {
846+ return err
847+ }
848+ }
849+
850+ for _, n := range ts.Indexes {
851+ if err := Walk(visit, n); err != nil {
852+ return err
853+ }
854+ }
855+
856+ return nil
857+}
858+
859+// ColumnDefinition describes a column in a CREATE TABLE statement
860+type ColumnDefinition struct {
861+ Name ColIdent
862+ Type ColumnType
863+}
864+
865+// Format formats the node.
866+func (col *ColumnDefinition) Format(buf *TrackedBuffer) {
867+ buf.Myprintf("%v %v", col.Name, &col.Type)
868+}
869+
870+func (col *ColumnDefinition) walkSubtree(visit Visit) error {
871+ if col == nil {
872+ return nil
873+ }
874+ return Walk(
875+ visit,
876+ col.Name,
877+ &col.Type,
878+ )
879+}
880+
881+// ColumnType represents a sql type in a CREATE TABLE statement
882+// All optional fields are nil if not specified
883+type ColumnType struct {
884+ // The base type string
885+ Type string
886+
887+ // Generic field options.
888+ NotNull BoolVal
889+ Autoincrement BoolVal
890+ Default *SQLVal
891+ OnUpdate *SQLVal
892+ Comment *SQLVal
893+
894+ // Numeric field options
895+ Length *SQLVal
896+ Unsigned BoolVal
897+ Zerofill BoolVal
898+ Scale *SQLVal
899+
900+ // Text field options
901+ Charset string
902+ Collate string
903+
904+ // Enum values
905+ EnumValues []string
906+
907+ // Key specification
908+ KeyOpt ColumnKeyOption
909+}
910+
911+// Format returns a canonical string representation of the type and all relevant options
912+func (ct *ColumnType) Format(buf *TrackedBuffer) {
913+ buf.Myprintf("%s", ct.Type)
914+
915+ if ct.Length != nil && ct.Scale != nil {
916+ buf.Myprintf("(%v,%v)", ct.Length, ct.Scale)
917+
918+ } else if ct.Length != nil {
919+ buf.Myprintf("(%v)", ct.Length)
920+ }
921+
922+ if ct.EnumValues != nil {
923+ buf.Myprintf("(%s)", strings.Join(ct.EnumValues, ", "))
924+ }
925+
926+ opts := make([]string, 0, 16)
927+ if ct.Unsigned {
928+ opts = append(opts, keywordStrings[UNSIGNED])
929+ }
930+ if ct.Zerofill {
931+ opts = append(opts, keywordStrings[ZEROFILL])
932+ }
933+ if ct.Charset != "" {
934+ opts = append(opts, keywordStrings[CHARACTER], keywordStrings[SET], ct.Charset)
935+ }
936+ if ct.Collate != "" {
937+ opts = append(opts, keywordStrings[COLLATE], ct.Collate)
938+ }
939+ if ct.NotNull {
940+ opts = append(opts, keywordStrings[NOT], keywordStrings[NULL])
941+ }
942+ if ct.Default != nil {
943+ opts = append(opts, keywordStrings[DEFAULT], String(ct.Default))
944+ }
945+ if ct.OnUpdate != nil {
946+ opts = append(opts, keywordStrings[ON], keywordStrings[UPDATE], String(ct.OnUpdate))
947+ }
948+ if ct.Autoincrement {
949+ opts = append(opts, keywordStrings[AUTO_INCREMENT])
950+ }
951+ if ct.Comment != nil {
952+ opts = append(opts, keywordStrings[COMMENT_KEYWORD], String(ct.Comment))
953+ }
954+ if ct.KeyOpt == colKeyPrimary {
955+ opts = append(opts, keywordStrings[PRIMARY], keywordStrings[KEY])
956+ }
957+ if ct.KeyOpt == colKeyUnique {
958+ opts = append(opts, keywordStrings[UNIQUE])
959+ }
960+ if ct.KeyOpt == colKeyUniqueKey {
961+ opts = append(opts, keywordStrings[UNIQUE], keywordStrings[KEY])
962+ }
963+ if ct.KeyOpt == colKeySpatialKey {
964+ opts = append(opts, keywordStrings[SPATIAL], keywordStrings[KEY])
965+ }
966+ if ct.KeyOpt == colKey {
967+ opts = append(opts, keywordStrings[KEY])
968+ }
969+
970+ if len(opts) != 0 {
971+ buf.Myprintf(" %s", strings.Join(opts, " "))
972+ }
973+}
974+
975+// DescribeType returns the abbreviated type information as required for
976+// describe table
977+func (ct *ColumnType) DescribeType() string {
978+ buf := NewTrackedBuffer(nil)
979+ buf.Myprintf("%s", ct.Type)
980+ if ct.Length != nil && ct.Scale != nil {
981+ buf.Myprintf("(%v,%v)", ct.Length, ct.Scale)
982+ } else if ct.Length != nil {
983+ buf.Myprintf("(%v)", ct.Length)
984+ }
985+
986+ opts := make([]string, 0, 16)
987+ if ct.Unsigned {
988+ opts = append(opts, keywordStrings[UNSIGNED])
989+ }
990+ if ct.Zerofill {
991+ opts = append(opts, keywordStrings[ZEROFILL])
992+ }
993+ if len(opts) != 0 {
994+ buf.Myprintf(" %s", strings.Join(opts, " "))
995+ }
996+ return buf.String()
997+}
998+
999+// SQLType returns the sqltypes type code for the given column
1000+func (ct *ColumnType) SQLType() querypb.Type {
1001+ switch ct.Type {
1002+ case keywordStrings[TINYINT]:
1003+ if ct.Unsigned {
1004+ return sqltypes.Uint8
1005+ }
1006+ return sqltypes.Int8
1007+ case keywordStrings[SMALLINT]:
1008+ if ct.Unsigned {
1009+ return sqltypes.Uint16
1010+ }
1011+ return sqltypes.Int16
1012+ case keywordStrings[MEDIUMINT]:
1013+ if ct.Unsigned {
1014+ return sqltypes.Uint24
1015+ }
1016+ return sqltypes.Int24
1017+ case keywordStrings[INT]:
1018+ fallthrough
1019+ case keywordStrings[INTEGER]:
1020+ if ct.Unsigned {
1021+ return sqltypes.Uint32
1022+ }
1023+ return sqltypes.Int32
1024+ case keywordStrings[BIGINT]:
1025+ if ct.Unsigned {
1026+ return sqltypes.Uint64
1027+ }
1028+ return sqltypes.Int64
1029+ case keywordStrings[TEXT]:
1030+ return sqltypes.Text
1031+ case keywordStrings[TINYTEXT]:
1032+ return sqltypes.Text
1033+ case keywordStrings[MEDIUMTEXT]:
1034+ return sqltypes.Text
1035+ case keywordStrings[LONGTEXT]:
1036+ return sqltypes.Text
1037+ case keywordStrings[BLOB]:
1038+ return sqltypes.Blob
1039+ case keywordStrings[TINYBLOB]:
1040+ return sqltypes.Blob
1041+ case keywordStrings[MEDIUMBLOB]:
1042+ return sqltypes.Blob
1043+ case keywordStrings[LONGBLOB]:
1044+ return sqltypes.Blob
1045+ case keywordStrings[CHAR]:
1046+ return sqltypes.Char
1047+ case keywordStrings[VARCHAR]:
1048+ return sqltypes.VarChar
1049+ case keywordStrings[BINARY]:
1050+ return sqltypes.Binary
1051+ case keywordStrings[VARBINARY]:
1052+ return sqltypes.VarBinary
1053+ case keywordStrings[DATE]:
1054+ return sqltypes.Date
1055+ case keywordStrings[TIME]:
1056+ return sqltypes.Time
1057+ case keywordStrings[DATETIME]:
1058+ return sqltypes.Datetime
1059+ case keywordStrings[TIMESTAMP]:
1060+ return sqltypes.Timestamp
1061+ case keywordStrings[YEAR]:
1062+ return sqltypes.Year
1063+ case keywordStrings[FLOAT_TYPE]:
1064+ return sqltypes.Float32
1065+ case keywordStrings[DOUBLE]:
1066+ return sqltypes.Float64
1067+ case keywordStrings[DECIMAL]:
1068+ return sqltypes.Decimal
1069+ case keywordStrings[BIT]:
1070+ return sqltypes.Bit
1071+ case keywordStrings[ENUM]:
1072+ return sqltypes.Enum
1073+ case keywordStrings[SET]:
1074+ return sqltypes.Set
1075+ case keywordStrings[JSON]:
1076+ return sqltypes.TypeJSON
1077+ case keywordStrings[GEOMETRY]:
1078+ return sqltypes.Geometry
1079+ case keywordStrings[POINT]:
1080+ return sqltypes.Geometry
1081+ case keywordStrings[LINESTRING]:
1082+ return sqltypes.Geometry
1083+ case keywordStrings[POLYGON]:
1084+ return sqltypes.Geometry
1085+ case keywordStrings[GEOMETRYCOLLECTION]:
1086+ return sqltypes.Geometry
1087+ case keywordStrings[MULTIPOINT]:
1088+ return sqltypes.Geometry
1089+ case keywordStrings[MULTILINESTRING]:
1090+ return sqltypes.Geometry
1091+ case keywordStrings[MULTIPOLYGON]:
1092+ return sqltypes.Geometry
1093+ }
1094+ panic("unimplemented type " + ct.Type)
1095+}
1096+
1097+func (ct *ColumnType) walkSubtree(visit Visit) error {
1098+ return nil
1099+}
1100+
1101+// IndexDefinition describes an index in a CREATE TABLE statement
1102+type IndexDefinition struct {
1103+ Info *IndexInfo
1104+ Columns []*IndexColumn
1105+ Options []*IndexOption
1106+}
1107+
1108+// Format formats the node.
1109+func (idx *IndexDefinition) Format(buf *TrackedBuffer) {
1110+ buf.Myprintf("%v (", idx.Info)
1111+ for i, col := range idx.Columns {
1112+ if i != 0 {
1113+ buf.Myprintf(", %v", col.Column)
1114+ } else {
1115+ buf.Myprintf("%v", col.Column)
1116+ }
1117+ if col.Length != nil {
1118+ buf.Myprintf("(%v)", col.Length)
1119+ }
1120+ }
1121+ buf.Myprintf(")")
1122+
1123+ for _, opt := range idx.Options {
1124+ buf.Myprintf(" %s", opt.Name)
1125+ if opt.Using != "" {
1126+ buf.Myprintf(" %s", opt.Using)
1127+ } else {
1128+ buf.Myprintf(" %v", opt.Value)
1129+ }
1130+ }
1131+}
1132+
1133+func (idx *IndexDefinition) walkSubtree(visit Visit) error {
1134+ if idx == nil {
1135+ return nil
1136+ }
1137+
1138+ for _, n := range idx.Columns {
1139+ if err := Walk(visit, n.Column); err != nil {
1140+ return err
1141+ }
1142+ }
1143+
1144+ return nil
1145+}
1146+
1147+// IndexInfo describes the name and type of an index in a CREATE TABLE statement
1148+type IndexInfo struct {
1149+ Type string
1150+ Name ColIdent
1151+ Primary bool
1152+ Spatial bool
1153+ Unique bool
1154+}
1155+
1156+// Format formats the node.
1157+func (ii *IndexInfo) Format(buf *TrackedBuffer) {
1158+ if ii.Primary {
1159+ buf.Myprintf("%s", ii.Type)
1160+ } else {
1161+ buf.Myprintf("%s %v", ii.Type, ii.Name)
1162+ }
1163+}
1164+
1165+func (ii *IndexInfo) walkSubtree(visit Visit) error {
1166+ return Walk(visit, ii.Name)
1167+}
1168+
1169+// IndexColumn describes a column in an index definition with optional length
1170+type IndexColumn struct {
1171+ Column ColIdent
1172+ Length *SQLVal
1173+}
1174+
1175+// LengthScaleOption is used for types that have an optional length
1176+// and scale
1177+type LengthScaleOption struct {
1178+ Length *SQLVal
1179+ Scale *SQLVal
1180+}
1181+
1182+// IndexOption is used for trailing options for indexes: COMMENT, KEY_BLOCK_SIZE, USING
1183+type IndexOption struct {
1184+ Name string
1185+ Value *SQLVal
1186+ Using string
1187+}
1188+
1189+// ColumnKeyOption indicates whether or not the given column is defined as an
1190+// index element and contains the type of the option
1191+type ColumnKeyOption int
1192+
1193+const (
1194+ colKeyNone ColumnKeyOption = iota
1195+ colKeyPrimary
1196+ colKeySpatialKey
1197+ colKeyUnique
1198+ colKeyUniqueKey
1199+ colKey
1200+)
1201+
1202+// VindexSpec defines a vindex for a CREATE VINDEX or DROP VINDEX statement
1203+type VindexSpec struct {
1204+ Name ColIdent
1205+ Type ColIdent
1206+ Params []VindexParam
1207+}
1208+
1209+// ParseParams parses the vindex parameter list, pulling out the special-case
1210+// "owner" parameter
1211+func (node *VindexSpec) ParseParams() (string, map[string]string) {
1212+ var owner string
1213+ params := map[string]string{}
1214+ for _, p := range node.Params {
1215+ if p.Key.Lowered() == VindexOwnerStr {
1216+ owner = p.Val
1217+ } else {
1218+ params[p.Key.String()] = p.Val
1219+ }
1220+ }
1221+ return owner, params
1222+}
1223+
1224+// Format formats the node. The "CREATE VINDEX" preamble was formatted in
1225+// the containing DDL node Format, so this just prints the type, any
1226+// parameters, and optionally the owner
1227+func (node *VindexSpec) Format(buf *TrackedBuffer) {
1228+ buf.Myprintf("using %v", node.Type)
1229+
1230+ numParams := len(node.Params)
1231+ if numParams != 0 {
1232+ buf.Myprintf(" with ")
1233+ for i, p := range node.Params {
1234+ if i != 0 {
1235+ buf.Myprintf(", ")
1236+ }
1237+ buf.Myprintf("%v", p)
1238+ }
1239+ }
1240+}
1241+
1242+func (node *VindexSpec) walkSubtree(visit Visit) error {
1243+ err := Walk(visit,
1244+ node.Name,
1245+ )
1246+
1247+ if err != nil {
1248+ return err
1249+ }
1250+
1251+ for _, p := range node.Params {
1252+ err := Walk(visit, p)
1253+
1254+ if err != nil {
1255+ return err
1256+ }
1257+ }
1258+ return nil
1259+}
1260+
1261+// VindexParam defines a key/value parameter for a CREATE VINDEX statement
1262+type VindexParam struct {
1263+ Key ColIdent
1264+ Val string
1265+}
1266+
1267+// Format formats the node.
1268+func (node VindexParam) Format(buf *TrackedBuffer) {
1269+ buf.Myprintf("%s=%s", node.Key.String(), node.Val)
1270+}
1271+
1272+func (node VindexParam) walkSubtree(visit Visit) error {
1273+ return Walk(visit,
1274+ node.Key,
1275+ )
1276+}
1277+
1278+// Show represents a show statement.
1279+type Show struct {
1280+ Type string
1281+ OnTable TableName
1282+ ShowTablesOpt *ShowTablesOpt
1283+ Scope string
1284+}
1285+
1286+// Format formats the node.
1287+func (node *Show) Format(buf *TrackedBuffer) {
1288+ if node.Type == "tables" && node.ShowTablesOpt != nil {
1289+ opt := node.ShowTablesOpt
1290+ if opt.DbName != "" {
1291+ if opt.Filter != nil {
1292+ buf.Myprintf("show %s%stables from %s %v", opt.Extended, opt.Full, opt.DbName, opt.Filter)
1293+ } else {
1294+ buf.Myprintf("show %s%stables from %s", opt.Extended, opt.Full, opt.DbName)
1295+ }
1296+ } else {
1297+ if opt.Filter != nil {
1298+ buf.Myprintf("show %s%stables %v", opt.Extended, opt.Full, opt.Filter)
1299+ } else {
1300+ buf.Myprintf("show %s%stables", opt.Extended, opt.Full)
1301+ }
1302+ }
1303+ return
1304+ }
1305+ if node.Scope == "" {
1306+ buf.Myprintf("show %s", node.Type)
1307+ } else {
1308+ buf.Myprintf("show %s %s", node.Scope, node.Type)
1309+ }
1310+ if node.HasOnTable() {
1311+ buf.Myprintf(" on %v", node.OnTable)
1312+ }
1313+}
1314+
1315+// HasOnTable returns true if the show statement has an "on" clause
1316+func (node *Show) HasOnTable() bool {
1317+ return node.OnTable.Name.v != ""
1318+}
1319+
1320+func (node *Show) walkSubtree(visit Visit) error {
1321+ return nil
1322+}
1323+
1324+// ShowTablesOpt is show tables option
1325+type ShowTablesOpt struct {
1326+ Extended string
1327+ Full string
1328+ DbName string
1329+ Filter *ShowFilter
1330+}
1331+
1332+// ShowFilter is show tables filter
1333+type ShowFilter struct {
1334+ Like string
1335+ Filter Expr
1336+}
1337+
1338+// Format formats the node.
1339+func (node *ShowFilter) Format(buf *TrackedBuffer) {
1340+ if node.Like != "" {
1341+ buf.Myprintf("like '%s'", node.Like)
1342+ } else {
1343+ buf.Myprintf("where %v", node.Filter)
1344+ }
1345+}
1346+
1347+func (node *ShowFilter) walkSubtree(visit Visit) error {
1348+ return nil
1349+}
1350+
1351+// Use represents a use statement.
1352+type Use struct {
1353+ DBName TableIdent
1354+}
1355+
1356+// Format formats the node.
1357+func (node *Use) Format(buf *TrackedBuffer) {
1358+ if node.DBName.v != "" {
1359+ buf.Myprintf("use %v", node.DBName)
1360+ } else {
1361+ buf.Myprintf("use")
1362+ }
1363+}
1364+
1365+func (node *Use) walkSubtree(visit Visit) error {
1366+ return Walk(visit, node.DBName)
1367+}
1368+
1369+// Begin represents a Begin statement.
1370+type Begin struct{}
1371+
1372+// Format formats the node.
1373+func (node *Begin) Format(buf *TrackedBuffer) {
1374+ buf.WriteString("begin")
1375+}
1376+
1377+func (node *Begin) walkSubtree(visit Visit) error {
1378+ return nil
1379+}
1380+
1381+// Commit represents a Commit statement.
1382+type Commit struct{}
1383+
1384+// Format formats the node.
1385+func (node *Commit) Format(buf *TrackedBuffer) {
1386+ buf.WriteString("commit")
1387+}
1388+
1389+func (node *Commit) walkSubtree(visit Visit) error {
1390+ return nil
1391+}
1392+
1393+// Rollback represents a Rollback statement.
1394+type Rollback struct{}
1395+
1396+// Format formats the node.
1397+func (node *Rollback) Format(buf *TrackedBuffer) {
1398+ buf.WriteString("rollback")
1399+}
1400+
1401+func (node *Rollback) walkSubtree(visit Visit) error {
1402+ return nil
1403+}
1404+
1405+// OtherRead represents a DESCRIBE, or EXPLAIN statement.
1406+// It should be used only as an indicator. It does not contain
1407+// the full AST for the statement.
1408+type OtherRead struct{}
1409+
1410+// Format formats the node.
1411+func (node *OtherRead) Format(buf *TrackedBuffer) {
1412+ buf.WriteString("otherread")
1413+}
1414+
1415+func (node *OtherRead) walkSubtree(visit Visit) error {
1416+ return nil
1417+}
1418+
1419+// OtherAdmin represents a misc statement that relies on ADMIN privileges,
1420+// such as REPAIR, OPTIMIZE, or TRUNCATE statement.
1421+// It should be used only as an indicator. It does not contain
1422+// the full AST for the statement.
1423+type OtherAdmin struct{}
1424+
1425+// Format formats the node.
1426+func (node *OtherAdmin) Format(buf *TrackedBuffer) {
1427+ buf.WriteString("otheradmin")
1428+}
1429+
1430+func (node *OtherAdmin) walkSubtree(visit Visit) error {
1431+ return nil
1432+}
1433+
1434+// Comments represents a list of comments.
1435+type Comments [][]byte
1436+
1437+// Format formats the node.
1438+func (node Comments) Format(buf *TrackedBuffer) {
1439+ for _, c := range node {
1440+ buf.Myprintf("%s ", c)
1441+ }
1442+}
1443+
1444+func (node Comments) walkSubtree(visit Visit) error {
1445+ return nil
1446+}
1447+
1448+// SelectExprs represents SELECT expressions.
1449+type SelectExprs []SelectExpr
1450+
1451+// Format formats the node.
1452+func (node SelectExprs) Format(buf *TrackedBuffer) {
1453+ var prefix string
1454+ for _, n := range node {
1455+ buf.Myprintf("%s%v", prefix, n)
1456+ prefix = ", "
1457+ }
1458+}
1459+
1460+func (node SelectExprs) walkSubtree(visit Visit) error {
1461+ for _, n := range node {
1462+ if err := Walk(visit, n); err != nil {
1463+ return err
1464+ }
1465+ }
1466+ return nil
1467+}
1468+
1469+// SelectExpr represents a SELECT expression.
1470+type SelectExpr interface {
1471+ iSelectExpr()
1472+ SQLNode
1473+}
1474+
1475+func (*StarExpr) iSelectExpr() {}
1476+func (*AliasedExpr) iSelectExpr() {}
1477+func (Nextval) iSelectExpr() {}
1478+
1479+// StarExpr defines a '*' or 'table.*' expression.
1480+type StarExpr struct {
1481+ TableName TableName
1482+}
1483+
1484+// Format formats the node.
1485+func (node *StarExpr) Format(buf *TrackedBuffer) {
1486+ if !node.TableName.IsEmpty() {
1487+ buf.Myprintf("%v.", node.TableName)
1488+ }
1489+ buf.Myprintf("*")
1490+}
1491+
1492+func (node *StarExpr) walkSubtree(visit Visit) error {
1493+ if node == nil {
1494+ return nil
1495+ }
1496+ return Walk(
1497+ visit,
1498+ node.TableName,
1499+ )
1500+}
1501+
1502+// AliasedExpr defines an aliased SELECT expression.
1503+type AliasedExpr struct {
1504+ Expr Expr
1505+ As ColIdent
1506+}
1507+
1508+// Format formats the node.
1509+func (node *AliasedExpr) Format(buf *TrackedBuffer) {
1510+ buf.Myprintf("%v", node.Expr)
1511+ if !node.As.IsEmpty() {
1512+ buf.Myprintf(" as %v", node.As)
1513+ }
1514+}
1515+
1516+func (node *AliasedExpr) walkSubtree(visit Visit) error {
1517+ if node == nil {
1518+ return nil
1519+ }
1520+ return Walk(
1521+ visit,
1522+ node.Expr,
1523+ node.As,
1524+ )
1525+}
1526+
1527+// Nextval defines the NEXT VALUE expression.
1528+type Nextval struct {
1529+ Expr Expr
1530+}
1531+
1532+// Format formats the node.
1533+func (node Nextval) Format(buf *TrackedBuffer) {
1534+ buf.Myprintf("next %v values", node.Expr)
1535+}
1536+
1537+func (node Nextval) walkSubtree(visit Visit) error {
1538+ return Walk(visit, node.Expr)
1539+}
1540+
1541+// Columns represents an insert column list.
1542+type Columns []ColIdent
1543+
1544+// Format formats the node.
1545+func (node Columns) Format(buf *TrackedBuffer) {
1546+ if node == nil {
1547+ return
1548+ }
1549+ prefix := "("
1550+ for _, n := range node {
1551+ buf.Myprintf("%s%v", prefix, n)
1552+ prefix = ", "
1553+ }
1554+ buf.WriteString(")")
1555+}
1556+
1557+func (node Columns) walkSubtree(visit Visit) error {
1558+ for _, n := range node {
1559+ if err := Walk(visit, n); err != nil {
1560+ return err
1561+ }
1562+ }
1563+ return nil
1564+}
1565+
1566+// FindColumn finds a column in the column list, returning
1567+// the index if it exists or -1 otherwise
1568+func (node Columns) FindColumn(col ColIdent) int {
1569+ for i, colName := range node {
1570+ if colName.Equal(col) {
1571+ return i
1572+ }
1573+ }
1574+ return -1
1575+}
1576+
1577+// Partitions is a type alias for Columns so we can handle printing efficiently
1578+type Partitions Columns
1579+
1580+// Format formats the node
1581+func (node Partitions) Format(buf *TrackedBuffer) {
1582+ if node == nil {
1583+ return
1584+ }
1585+ prefix := " partition ("
1586+ for _, n := range node {
1587+ buf.Myprintf("%s%v", prefix, n)
1588+ prefix = ", "
1589+ }
1590+ buf.WriteString(")")
1591+}
1592+
1593+func (node Partitions) walkSubtree(visit Visit) error {
1594+ for _, n := range node {
1595+ if err := Walk(visit, n); err != nil {
1596+ return err
1597+ }
1598+ }
1599+ return nil
1600+}
1601+
1602+// TableExprs represents a list of table expressions.
1603+type TableExprs []TableExpr
1604+
1605+// Format formats the node.
1606+func (node TableExprs) Format(buf *TrackedBuffer) {
1607+ var prefix string
1608+ for _, n := range node {
1609+ buf.Myprintf("%s%v", prefix, n)
1610+ prefix = ", "
1611+ }
1612+}
1613+
1614+func (node TableExprs) walkSubtree(visit Visit) error {
1615+ for _, n := range node {
1616+ if err := Walk(visit, n); err != nil {
1617+ return err
1618+ }
1619+ }
1620+ return nil
1621+}
1622+
1623+// TableExpr represents a table expression.
1624+type TableExpr interface {
1625+ iTableExpr()
1626+ SQLNode
1627+}
1628+
1629+func (*AliasedTableExpr) iTableExpr() {}
1630+func (*ParenTableExpr) iTableExpr() {}
1631+func (*JoinTableExpr) iTableExpr() {}
1632+
1633+// AliasedTableExpr represents a table expression
1634+// coupled with an optional alias or index hint.
1635+// If As is empty, no alias was used.
1636+type AliasedTableExpr struct {
1637+ Expr SimpleTableExpr
1638+ Partitions Partitions
1639+ As TableIdent
1640+ Hints *IndexHints
1641+}
1642+
1643+// Format formats the node.
1644+func (node *AliasedTableExpr) Format(buf *TrackedBuffer) {
1645+ buf.Myprintf("%v%v", node.Expr, node.Partitions)
1646+ if !node.As.IsEmpty() {
1647+ buf.Myprintf(" as %v", node.As)
1648+ }
1649+ if node.Hints != nil {
1650+ // Hint node provides the space padding.
1651+ buf.Myprintf("%v", node.Hints)
1652+ }
1653+}
1654+
1655+func (node *AliasedTableExpr) walkSubtree(visit Visit) error {
1656+ if node == nil {
1657+ return nil
1658+ }
1659+ return Walk(
1660+ visit,
1661+ node.Expr,
1662+ node.As,
1663+ node.Hints,
1664+ )
1665+}
1666+
1667+// RemoveHints returns a new AliasedTableExpr with the hints removed.
1668+func (node *AliasedTableExpr) RemoveHints() *AliasedTableExpr {
1669+ noHints := *node
1670+ noHints.Hints = nil
1671+ return &noHints
1672+}
1673+
1674+// SimpleTableExpr represents a simple table expression.
1675+type SimpleTableExpr interface {
1676+ iSimpleTableExpr()
1677+ SQLNode
1678+}
1679+
1680+func (TableName) iSimpleTableExpr() {}
1681+func (*Subquery) iSimpleTableExpr() {}
1682+
1683+// TableNames is a list of TableName.
1684+type TableNames []TableName
1685+
1686+// Format formats the node.
1687+func (node TableNames) Format(buf *TrackedBuffer) {
1688+ var prefix string
1689+ for _, n := range node {
1690+ buf.Myprintf("%s%v", prefix, n)
1691+ prefix = ", "
1692+ }
1693+}
1694+
1695+func (node TableNames) walkSubtree(visit Visit) error {
1696+ for _, n := range node {
1697+ if err := Walk(visit, n); err != nil {
1698+ return err
1699+ }
1700+ }
1701+ return nil
1702+}
1703+
1704+// TableName represents a table name.
1705+// Qualifier, if specified, represents a database or keyspace.
1706+// TableName is a value struct whose fields are case sensitive.
1707+// This means two TableName vars can be compared for equality
1708+// and a TableName can also be used as key in a map.
1709+type TableName struct {
1710+ Name, Qualifier TableIdent
1711+}
1712+
1713+// Format formats the node.
1714+func (node TableName) Format(buf *TrackedBuffer) {
1715+ if node.IsEmpty() {
1716+ return
1717+ }
1718+ if !node.Qualifier.IsEmpty() {
1719+ buf.Myprintf("%v.", node.Qualifier)
1720+ }
1721+ buf.Myprintf("%v", node.Name)
1722+}
1723+
1724+func (node TableName) walkSubtree(visit Visit) error {
1725+ return Walk(
1726+ visit,
1727+ node.Name,
1728+ node.Qualifier,
1729+ )
1730+}
1731+
1732+// IsEmpty returns true if TableName is nil or empty.
1733+func (node TableName) IsEmpty() bool {
1734+ // If Name is empty, Qualifer is also empty.
1735+ return node.Name.IsEmpty()
1736+}
1737+
1738+// ToViewName returns a TableName acceptable for use as a VIEW. VIEW names are
1739+// always lowercase, so ToViewName lowercasese the name. Databases are case-sensitive
1740+// so Qualifier is left untouched.
1741+func (node TableName) ToViewName() TableName {
1742+ return TableName{
1743+ Qualifier: node.Qualifier,
1744+ Name: NewTableIdent(strings.ToLower(node.Name.v)),
1745+ }
1746+}
1747+
1748+// ParenTableExpr represents a parenthesized list of TableExpr.
1749+type ParenTableExpr struct {
1750+ Exprs TableExprs
1751+}
1752+
1753+// Format formats the node.
1754+func (node *ParenTableExpr) Format(buf *TrackedBuffer) {
1755+ buf.Myprintf("(%v)", node.Exprs)
1756+}
1757+
1758+func (node *ParenTableExpr) walkSubtree(visit Visit) error {
1759+ if node == nil {
1760+ return nil
1761+ }
1762+ return Walk(
1763+ visit,
1764+ node.Exprs,
1765+ )
1766+}
1767+
1768+// JoinCondition represents the join conditions (either a ON or USING clause)
1769+// of a JoinTableExpr.
1770+type JoinCondition struct {
1771+ On Expr
1772+ Using Columns
1773+}
1774+
1775+// Format formats the node.
1776+func (node JoinCondition) Format(buf *TrackedBuffer) {
1777+ if node.On != nil {
1778+ buf.Myprintf(" on %v", node.On)
1779+ }
1780+ if node.Using != nil {
1781+ buf.Myprintf(" using %v", node.Using)
1782+ }
1783+}
1784+
1785+func (node JoinCondition) walkSubtree(visit Visit) error {
1786+ return Walk(
1787+ visit,
1788+ node.On,
1789+ node.Using,
1790+ )
1791+}
1792+
1793+// JoinTableExpr represents a TableExpr that's a JOIN operation.
1794+type JoinTableExpr struct {
1795+ LeftExpr TableExpr
1796+ Join string
1797+ RightExpr TableExpr
1798+ Condition JoinCondition
1799+}
1800+
1801+// JoinTableExpr.Join
1802+const (
1803+ JoinStr = "join"
1804+ StraightJoinStr = "straight_join"
1805+ LeftJoinStr = "left join"
1806+ RightJoinStr = "right join"
1807+ NaturalJoinStr = "natural join"
1808+ NaturalLeftJoinStr = "natural left join"
1809+ NaturalRightJoinStr = "natural right join"
1810+)
1811+
1812+// Format formats the node.
1813+func (node *JoinTableExpr) Format(buf *TrackedBuffer) {
1814+ buf.Myprintf("%v %s %v%v", node.LeftExpr, node.Join, node.RightExpr, node.Condition)
1815+}
1816+
1817+func (node *JoinTableExpr) walkSubtree(visit Visit) error {
1818+ if node == nil {
1819+ return nil
1820+ }
1821+ return Walk(
1822+ visit,
1823+ node.LeftExpr,
1824+ node.RightExpr,
1825+ node.Condition,
1826+ )
1827+}
1828+
1829+// IndexHints represents a list of index hints.
1830+type IndexHints struct {
1831+ Type string
1832+ Indexes []ColIdent
1833+}
1834+
1835+// Index hints.
1836+const (
1837+ UseStr = "use "
1838+ IgnoreStr = "ignore "
1839+ ForceStr = "force "
1840+)
1841+
1842+// Format formats the node.
1843+func (node *IndexHints) Format(buf *TrackedBuffer) {
1844+ buf.Myprintf(" %sindex ", node.Type)
1845+ prefix := "("
1846+ for _, n := range node.Indexes {
1847+ buf.Myprintf("%s%v", prefix, n)
1848+ prefix = ", "
1849+ }
1850+ buf.Myprintf(")")
1851+}
1852+
1853+func (node *IndexHints) walkSubtree(visit Visit) error {
1854+ if node == nil {
1855+ return nil
1856+ }
1857+ for _, n := range node.Indexes {
1858+ if err := Walk(visit, n); err != nil {
1859+ return err
1860+ }
1861+ }
1862+ return nil
1863+}
1864+
1865+// Where represents a WHERE or HAVING clause.
1866+type Where struct {
1867+ Type string
1868+ Expr Expr
1869+}
1870+
1871+// Where.Type
1872+const (
1873+ WhereStr = "where"
1874+ HavingStr = "having"
1875+)
1876+
1877+// NewWhere creates a WHERE or HAVING clause out
1878+// of a Expr. If the expression is nil, it returns nil.
1879+func NewWhere(typ string, expr Expr) *Where {
1880+ if expr == nil {
1881+ return nil
1882+ }
1883+ return &Where{Type: typ, Expr: expr}
1884+}
1885+
1886+// Format formats the node.
1887+func (node *Where) Format(buf *TrackedBuffer) {
1888+ if node == nil || node.Expr == nil {
1889+ return
1890+ }
1891+ buf.Myprintf(" %s %v", node.Type, node.Expr)
1892+}
1893+
1894+func (node *Where) walkSubtree(visit Visit) error {
1895+ if node == nil {
1896+ return nil
1897+ }
1898+ return Walk(
1899+ visit,
1900+ node.Expr,
1901+ )
1902+}
1903+
1904+// Expr represents an expression.
1905+type Expr interface {
1906+ iExpr()
1907+ // replace replaces any subexpression that matches
1908+ // from with to. The implementation can use the
1909+ // replaceExprs convenience function.
1910+ replace(from, to Expr) bool
1911+ SQLNode
1912+}
1913+
1914+func (*AndExpr) iExpr() {}
1915+func (*OrExpr) iExpr() {}
1916+func (*NotExpr) iExpr() {}
1917+func (*ParenExpr) iExpr() {}
1918+func (*ComparisonExpr) iExpr() {}
1919+func (*RangeCond) iExpr() {}
1920+func (*IsExpr) iExpr() {}
1921+func (*ExistsExpr) iExpr() {}
1922+func (*SQLVal) iExpr() {}
1923+func (*NullVal) iExpr() {}
1924+func (BoolVal) iExpr() {}
1925+func (*ColName) iExpr() {}
1926+func (ValTuple) iExpr() {}
1927+func (*Subquery) iExpr() {}
1928+func (ListArg) iExpr() {}
1929+func (*BinaryExpr) iExpr() {}
1930+func (*UnaryExpr) iExpr() {}
1931+func (*IntervalExpr) iExpr() {}
1932+func (*CollateExpr) iExpr() {}
1933+func (*FuncExpr) iExpr() {}
1934+func (*CaseExpr) iExpr() {}
1935+func (*ValuesFuncExpr) iExpr() {}
1936+func (*ConvertExpr) iExpr() {}
1937+func (*SubstrExpr) iExpr() {}
1938+func (*ConvertUsingExpr) iExpr() {}
1939+func (*MatchExpr) iExpr() {}
1940+func (*GroupConcatExpr) iExpr() {}
1941+func (*Default) iExpr() {}
1942+
1943+// ReplaceExpr finds the from expression from root
1944+// and replaces it with to. If from matches root,
1945+// then to is returned.
1946+func ReplaceExpr(root, from, to Expr) Expr {
1947+ if root == from {
1948+ return to
1949+ }
1950+ root.replace(from, to)
1951+ return root
1952+}
1953+
1954+// replaceExprs is a convenience function used by implementors
1955+// of the replace method.
1956+func replaceExprs(from, to Expr, exprs ...*Expr) bool {
1957+ for _, expr := range exprs {
1958+ if *expr == nil {
1959+ continue
1960+ }
1961+ if *expr == from {
1962+ *expr = to
1963+ return true
1964+ }
1965+ if (*expr).replace(from, to) {
1966+ return true
1967+ }
1968+ }
1969+ return false
1970+}
1971+
1972+// Exprs represents a list of value expressions.
1973+// It's not a valid expression because it's not parenthesized.
1974+type Exprs []Expr
1975+
1976+// Format formats the node.
1977+func (node Exprs) Format(buf *TrackedBuffer) {
1978+ var prefix string
1979+ for _, n := range node {
1980+ buf.Myprintf("%s%v", prefix, n)
1981+ prefix = ", "
1982+ }
1983+}
1984+
1985+func (node Exprs) walkSubtree(visit Visit) error {
1986+ for _, n := range node {
1987+ if err := Walk(visit, n); err != nil {
1988+ return err
1989+ }
1990+ }
1991+ return nil
1992+}
1993+
1994+// AndExpr represents an AND expression.
1995+type AndExpr struct {
1996+ Left, Right Expr
1997+}
1998+
1999+// Format formats the node.
2000+func (node *AndExpr) Format(buf *TrackedBuffer) {
2001+ buf.Myprintf("%v and %v", node.Left, node.Right)
2002+}
2003+
2004+func (node *AndExpr) walkSubtree(visit Visit) error {
2005+ if node == nil {
2006+ return nil
2007+ }
2008+ return Walk(
2009+ visit,
2010+ node.Left,
2011+ node.Right,
2012+ )
2013+}
2014+
2015+func (node *AndExpr) replace(from, to Expr) bool {
2016+ return replaceExprs(from, to, &node.Left, &node.Right)
2017+}
2018+
2019+// OrExpr represents an OR expression.
2020+type OrExpr struct {
2021+ Left, Right Expr
2022+}
2023+
2024+// Format formats the node.
2025+func (node *OrExpr) Format(buf *TrackedBuffer) {
2026+ buf.Myprintf("%v or %v", node.Left, node.Right)
2027+}
2028+
2029+func (node *OrExpr) walkSubtree(visit Visit) error {
2030+ if node == nil {
2031+ return nil
2032+ }
2033+ return Walk(
2034+ visit,
2035+ node.Left,
2036+ node.Right,
2037+ )
2038+}
2039+
2040+func (node *OrExpr) replace(from, to Expr) bool {
2041+ return replaceExprs(from, to, &node.Left, &node.Right)
2042+}
2043+
2044+// NotExpr represents a NOT expression.
2045+type NotExpr struct {
2046+ Expr Expr
2047+}
2048+
2049+// Format formats the node.
2050+func (node *NotExpr) Format(buf *TrackedBuffer) {
2051+ buf.Myprintf("not %v", node.Expr)
2052+}
2053+
2054+func (node *NotExpr) walkSubtree(visit Visit) error {
2055+ if node == nil {
2056+ return nil
2057+ }
2058+ return Walk(
2059+ visit,
2060+ node.Expr,
2061+ )
2062+}
2063+
2064+func (node *NotExpr) replace(from, to Expr) bool {
2065+ return replaceExprs(from, to, &node.Expr)
2066+}
2067+
2068+// ParenExpr represents a parenthesized boolean expression.
2069+type ParenExpr struct {
2070+ Expr Expr
2071+}
2072+
2073+// Format formats the node.
2074+func (node *ParenExpr) Format(buf *TrackedBuffer) {
2075+ buf.Myprintf("(%v)", node.Expr)
2076+}
2077+
2078+func (node *ParenExpr) walkSubtree(visit Visit) error {
2079+ if node == nil {
2080+ return nil
2081+ }
2082+ return Walk(
2083+ visit,
2084+ node.Expr,
2085+ )
2086+}
2087+
2088+func (node *ParenExpr) replace(from, to Expr) bool {
2089+ return replaceExprs(from, to, &node.Expr)
2090+}
2091+
2092+// ComparisonExpr represents a two-value comparison expression.
2093+type ComparisonExpr struct {
2094+ Operator string
2095+ Left, Right Expr
2096+ Escape Expr
2097+}
2098+
2099+// ComparisonExpr.Operator
2100+const (
2101+ EqualStr = "="
2102+ LessThanStr = "<"
2103+ GreaterThanStr = ">"
2104+ LessEqualStr = "<="
2105+ GreaterEqualStr = ">="
2106+ NotEqualStr = "!="
2107+ NullSafeEqualStr = "<=>"
2108+ InStr = "in"
2109+ NotInStr = "not in"
2110+ LikeStr = "like"
2111+ NotLikeStr = "not like"
2112+ RegexpStr = "regexp"
2113+ NotRegexpStr = "not regexp"
2114+ JSONExtractOp = "->"
2115+ JSONUnquoteExtractOp = "->>"
2116+)
2117+
2118+// Format formats the node.
2119+func (node *ComparisonExpr) Format(buf *TrackedBuffer) {
2120+ buf.Myprintf("%v %s %v", node.Left, node.Operator, node.Right)
2121+ if node.Escape != nil {
2122+ buf.Myprintf(" escape %v", node.Escape)
2123+ }
2124+}
2125+
2126+func (node *ComparisonExpr) walkSubtree(visit Visit) error {
2127+ if node == nil {
2128+ return nil
2129+ }
2130+ return Walk(
2131+ visit,
2132+ node.Left,
2133+ node.Right,
2134+ node.Escape,
2135+ )
2136+}
2137+
2138+func (node *ComparisonExpr) replace(from, to Expr) bool {
2139+ return replaceExprs(from, to, &node.Left, &node.Right, &node.Escape)
2140+}
2141+
2142+// RangeCond represents a BETWEEN or a NOT BETWEEN expression.
2143+type RangeCond struct {
2144+ Operator string
2145+ Left Expr
2146+ From, To Expr
2147+}
2148+
2149+// RangeCond.Operator
2150+const (
2151+ BetweenStr = "between"
2152+ NotBetweenStr = "not between"
2153+)
2154+
2155+// Format formats the node.
2156+func (node *RangeCond) Format(buf *TrackedBuffer) {
2157+ buf.Myprintf("%v %s %v and %v", node.Left, node.Operator, node.From, node.To)
2158+}
2159+
2160+func (node *RangeCond) walkSubtree(visit Visit) error {
2161+ if node == nil {
2162+ return nil
2163+ }
2164+ return Walk(
2165+ visit,
2166+ node.Left,
2167+ node.From,
2168+ node.To,
2169+ )
2170+}
2171+
2172+func (node *RangeCond) replace(from, to Expr) bool {
2173+ return replaceExprs(from, to, &node.Left, &node.From, &node.To)
2174+}
2175+
2176+// IsExpr represents an IS ... or an IS NOT ... expression.
2177+type IsExpr struct {
2178+ Operator string
2179+ Expr Expr
2180+}
2181+
2182+// IsExpr.Operator
2183+const (
2184+ IsNullStr = "is null"
2185+ IsNotNullStr = "is not null"
2186+ IsTrueStr = "is true"
2187+ IsNotTrueStr = "is not true"
2188+ IsFalseStr = "is false"
2189+ IsNotFalseStr = "is not false"
2190+)
2191+
2192+// Format formats the node.
2193+func (node *IsExpr) Format(buf *TrackedBuffer) {
2194+ buf.Myprintf("%v %s", node.Expr, node.Operator)
2195+}
2196+
2197+func (node *IsExpr) walkSubtree(visit Visit) error {
2198+ if node == nil {
2199+ return nil
2200+ }
2201+ return Walk(
2202+ visit,
2203+ node.Expr,
2204+ )
2205+}
2206+
2207+func (node *IsExpr) replace(from, to Expr) bool {
2208+ return replaceExprs(from, to, &node.Expr)
2209+}
2210+
2211+// ExistsExpr represents an EXISTS expression.
2212+type ExistsExpr struct {
2213+ Subquery *Subquery
2214+}
2215+
2216+// Format formats the node.
2217+func (node *ExistsExpr) Format(buf *TrackedBuffer) {
2218+ buf.Myprintf("exists %v", node.Subquery)
2219+}
2220+
2221+func (node *ExistsExpr) walkSubtree(visit Visit) error {
2222+ if node == nil {
2223+ return nil
2224+ }
2225+ return Walk(
2226+ visit,
2227+ node.Subquery,
2228+ )
2229+}
2230+
2231+func (node *ExistsExpr) replace(from, to Expr) bool {
2232+ return false
2233+}
2234+
2235+// ExprFromValue converts the given Value into an Expr or returns an error.
2236+func ExprFromValue(value sqltypes.Value) (Expr, error) {
2237+ // The type checks here follow the rules defined in sqltypes/types.go.
2238+ switch {
2239+ case value.Type() == sqltypes.Null:
2240+ return &NullVal{}, nil
2241+ case value.IsIntegral():
2242+ return NewIntVal(value.ToBytes()), nil
2243+ case value.IsFloat() || value.Type() == sqltypes.Decimal:
2244+ return NewFloatVal(value.ToBytes()), nil
2245+ case value.IsQuoted():
2246+ return NewStrVal(value.ToBytes()), nil
2247+ default:
2248+ // We cannot support sqltypes.Expression, or any other invalid type.
2249+ return nil, fmt.Errorf("cannot convert value %v to AST", value)
2250+ }
2251+}
2252+
2253+// ValType specifies the type for SQLVal.
2254+type ValType int
2255+
2256+// These are the possible Valtype values.
2257+// HexNum represents a 0x... value. It cannot
2258+// be treated as a simple value because it can
2259+// be interpreted differently depending on the
2260+// context.
2261+const (
2262+ StrVal = ValType(iota)
2263+ IntVal
2264+ FloatVal
2265+ HexNum
2266+ HexVal
2267+ ValArg
2268+ BitVal
2269+)
2270+
2271+// SQLVal represents a single value.
2272+type SQLVal struct {
2273+ Type ValType
2274+ Val []byte
2275+}
2276+
2277+// NewStrVal builds a new StrVal.
2278+func NewStrVal(in []byte) *SQLVal {
2279+ return &SQLVal{Type: StrVal, Val: in}
2280+}
2281+
2282+// NewIntVal builds a new IntVal.
2283+func NewIntVal(in []byte) *SQLVal {
2284+ return &SQLVal{Type: IntVal, Val: in}
2285+}
2286+
2287+// NewFloatVal builds a new FloatVal.
2288+func NewFloatVal(in []byte) *SQLVal {
2289+ return &SQLVal{Type: FloatVal, Val: in}
2290+}
2291+
2292+// NewHexNum builds a new HexNum.
2293+func NewHexNum(in []byte) *SQLVal {
2294+ return &SQLVal{Type: HexNum, Val: in}
2295+}
2296+
2297+// NewHexVal builds a new HexVal.
2298+func NewHexVal(in []byte) *SQLVal {
2299+ return &SQLVal{Type: HexVal, Val: in}
2300+}
2301+
2302+// NewBitVal builds a new BitVal containing a bit literal.
2303+func NewBitVal(in []byte) *SQLVal {
2304+ return &SQLVal{Type: BitVal, Val: in}
2305+}
2306+
2307+// NewValArg builds a new ValArg.
2308+func NewValArg(in []byte) *SQLVal {
2309+ return &SQLVal{Type: ValArg, Val: in}
2310+}
2311+
2312+// Format formats the node.
2313+func (node *SQLVal) Format(buf *TrackedBuffer) {
2314+ switch node.Type {
2315+ case StrVal:
2316+ sqltypes.MakeTrusted(sqltypes.VarBinary, node.Val).EncodeSQL(buf)
2317+ case IntVal, FloatVal, HexNum:
2318+ buf.Myprintf("%s", []byte(node.Val))
2319+ case HexVal:
2320+ buf.Myprintf("X'%s'", []byte(node.Val))
2321+ case BitVal:
2322+ buf.Myprintf("B'%s'", []byte(node.Val))
2323+ case ValArg:
2324+ buf.WriteArg(string(node.Val))
2325+ default:
2326+ panic("unexpected")
2327+ }
2328+}
2329+
2330+func (node *SQLVal) walkSubtree(visit Visit) error {
2331+ return nil
2332+}
2333+
2334+func (node *SQLVal) replace(from, to Expr) bool {
2335+ return false
2336+}
2337+
2338+// HexDecode decodes the hexval into bytes.
2339+func (node *SQLVal) HexDecode() ([]byte, error) {
2340+ dst := make([]byte, hex.DecodedLen(len([]byte(node.Val))))
2341+ _, err := hex.Decode(dst, []byte(node.Val))
2342+ if err != nil {
2343+ return nil, err
2344+ }
2345+ return dst, err
2346+}
2347+
2348+// NullVal represents a NULL value.
2349+type NullVal struct{}
2350+
2351+// Format formats the node.
2352+func (node *NullVal) Format(buf *TrackedBuffer) {
2353+ buf.Myprintf("null")
2354+}
2355+
2356+func (node *NullVal) walkSubtree(visit Visit) error {
2357+ return nil
2358+}
2359+
2360+func (node *NullVal) replace(from, to Expr) bool {
2361+ return false
2362+}
2363+
2364+// BoolVal is true or false.
2365+type BoolVal bool
2366+
2367+// Format formats the node.
2368+func (node BoolVal) Format(buf *TrackedBuffer) {
2369+ if node {
2370+ buf.Myprintf("true")
2371+ } else {
2372+ buf.Myprintf("false")
2373+ }
2374+}
2375+
2376+func (node BoolVal) walkSubtree(visit Visit) error {
2377+ return nil
2378+}
2379+
2380+func (node BoolVal) replace(from, to Expr) bool {
2381+ return false
2382+}
2383+
2384+// ColName represents a column name.
2385+type ColName struct {
2386+ // Metadata is not populated by the parser.
2387+ // It's a placeholder for analyzers to store
2388+ // additional data, typically info about which
2389+ // table or column this node references.
2390+ Metadata interface{}
2391+ Name ColIdent
2392+ Qualifier TableName
2393+}
2394+
2395+// Format formats the node.
2396+func (node *ColName) Format(buf *TrackedBuffer) {
2397+ if !node.Qualifier.IsEmpty() {
2398+ buf.Myprintf("%v.", node.Qualifier)
2399+ }
2400+ buf.Myprintf("%v", node.Name)
2401+}
2402+
2403+func (node *ColName) walkSubtree(visit Visit) error {
2404+ if node == nil {
2405+ return nil
2406+ }
2407+ return Walk(
2408+ visit,
2409+ node.Name,
2410+ node.Qualifier,
2411+ )
2412+}
2413+
2414+func (node *ColName) replace(from, to Expr) bool {
2415+ return false
2416+}
2417+
2418+// Equal returns true if the column names match.
2419+func (node *ColName) Equal(c *ColName) bool {
2420+ // Failsafe: ColName should not be empty.
2421+ if node == nil || c == nil {
2422+ return false
2423+ }
2424+ return node.Name.Equal(c.Name) && node.Qualifier == c.Qualifier
2425+}
2426+
2427+// ColTuple represents a list of column values.
2428+// It can be ValTuple, Subquery, ListArg.
2429+type ColTuple interface {
2430+ iColTuple()
2431+ Expr
2432+}
2433+
2434+func (ValTuple) iColTuple() {}
2435+func (*Subquery) iColTuple() {}
2436+func (ListArg) iColTuple() {}
2437+
2438+// ValTuple represents a tuple of actual values.
2439+type ValTuple Exprs
2440+
2441+// Format formats the node.
2442+func (node ValTuple) Format(buf *TrackedBuffer) {
2443+ buf.Myprintf("(%v)", Exprs(node))
2444+}
2445+
2446+func (node ValTuple) walkSubtree(visit Visit) error {
2447+ return Walk(visit, Exprs(node))
2448+}
2449+
2450+func (node ValTuple) replace(from, to Expr) bool {
2451+ for i := range node {
2452+ if replaceExprs(from, to, &node[i]) {
2453+ return true
2454+ }
2455+ }
2456+ return false
2457+}
2458+
2459+// Subquery represents a subquery.
2460+type Subquery struct {
2461+ Select SelectStatement
2462+}
2463+
2464+// Format formats the node.
2465+func (node *Subquery) Format(buf *TrackedBuffer) {
2466+ buf.Myprintf("(%v)", node.Select)
2467+}
2468+
2469+func (node *Subquery) walkSubtree(visit Visit) error {
2470+ if node == nil {
2471+ return nil
2472+ }
2473+ return Walk(
2474+ visit,
2475+ node.Select,
2476+ )
2477+}
2478+
2479+func (node *Subquery) replace(from, to Expr) bool {
2480+ return false
2481+}
2482+
2483+// ListArg represents a named list argument.
2484+type ListArg []byte
2485+
2486+// Format formats the node.
2487+func (node ListArg) Format(buf *TrackedBuffer) {
2488+ buf.WriteArg(string(node))
2489+}
2490+
2491+func (node ListArg) walkSubtree(visit Visit) error {
2492+ return nil
2493+}
2494+
2495+func (node ListArg) replace(from, to Expr) bool {
2496+ return false
2497+}
2498+
2499+// BinaryExpr represents a binary value expression.
2500+type BinaryExpr struct {
2501+ Operator string
2502+ Left, Right Expr
2503+}
2504+
2505+// BinaryExpr.Operator
2506+const (
2507+ BitAndStr = "&"
2508+ BitOrStr = "|"
2509+ BitXorStr = "^"
2510+ PlusStr = "+"
2511+ MinusStr = "-"
2512+ MultStr = "*"
2513+ DivStr = "/"
2514+ IntDivStr = "div"
2515+ ModStr = "%"
2516+ ShiftLeftStr = "<<"
2517+ ShiftRightStr = ">>"
2518+)
2519+
2520+// Format formats the node.
2521+func (node *BinaryExpr) Format(buf *TrackedBuffer) {
2522+ buf.Myprintf("%v %s %v", node.Left, node.Operator, node.Right)
2523+}
2524+
2525+func (node *BinaryExpr) walkSubtree(visit Visit) error {
2526+ if node == nil {
2527+ return nil
2528+ }
2529+ return Walk(
2530+ visit,
2531+ node.Left,
2532+ node.Right,
2533+ )
2534+}
2535+
2536+func (node *BinaryExpr) replace(from, to Expr) bool {
2537+ return replaceExprs(from, to, &node.Left, &node.Right)
2538+}
2539+
2540+// UnaryExpr represents a unary value expression.
2541+type UnaryExpr struct {
2542+ Operator string
2543+ Expr Expr
2544+}
2545+
2546+// UnaryExpr.Operator
2547+const (
2548+ UPlusStr = "+"
2549+ UMinusStr = "-"
2550+ TildaStr = "~"
2551+ BangStr = "!"
2552+ BinaryStr = "binary "
2553+ UBinaryStr = "_binary "
2554+)
2555+
2556+// Format formats the node.
2557+func (node *UnaryExpr) Format(buf *TrackedBuffer) {
2558+ if _, unary := node.Expr.(*UnaryExpr); unary {
2559+ buf.Myprintf("%s %v", node.Operator, node.Expr)
2560+ return
2561+ }
2562+ buf.Myprintf("%s%v", node.Operator, node.Expr)
2563+}
2564+
2565+func (node *UnaryExpr) walkSubtree(visit Visit) error {
2566+ if node == nil {
2567+ return nil
2568+ }
2569+ return Walk(
2570+ visit,
2571+ node.Expr,
2572+ )
2573+}
2574+
2575+func (node *UnaryExpr) replace(from, to Expr) bool {
2576+ return replaceExprs(from, to, &node.Expr)
2577+}
2578+
2579+// IntervalExpr represents a date-time INTERVAL expression.
2580+type IntervalExpr struct {
2581+ Expr Expr
2582+ Unit string
2583+}
2584+
2585+// Format formats the node.
2586+func (node *IntervalExpr) Format(buf *TrackedBuffer) {
2587+ buf.Myprintf("interval %v %s", node.Expr, node.Unit)
2588+}
2589+
2590+func (node *IntervalExpr) walkSubtree(visit Visit) error {
2591+ if node == nil {
2592+ return nil
2593+ }
2594+ return Walk(
2595+ visit,
2596+ node.Expr,
2597+ )
2598+}
2599+
2600+func (node *IntervalExpr) replace(from, to Expr) bool {
2601+ return replaceExprs(from, to, &node.Expr)
2602+}
2603+
2604+// CollateExpr represents dynamic collate operator.
2605+type CollateExpr struct {
2606+ Expr Expr
2607+ Charset string
2608+}
2609+
2610+// Format formats the node.
2611+func (node *CollateExpr) Format(buf *TrackedBuffer) {
2612+ buf.Myprintf("%v collate %s", node.Expr, node.Charset)
2613+}
2614+
2615+func (node *CollateExpr) walkSubtree(visit Visit) error {
2616+ if node == nil {
2617+ return nil
2618+ }
2619+ return Walk(
2620+ visit,
2621+ node.Expr,
2622+ )
2623+}
2624+
2625+func (node *CollateExpr) replace(from, to Expr) bool {
2626+ return replaceExprs(from, to, &node.Expr)
2627+}
2628+
2629+// FuncExpr represents a function call.
2630+type FuncExpr struct {
2631+ Qualifier TableIdent
2632+ Name ColIdent
2633+ Distinct bool
2634+ Exprs SelectExprs
2635+}
2636+
2637+// Format formats the node.
2638+func (node *FuncExpr) Format(buf *TrackedBuffer) {
2639+ var distinct string
2640+ if node.Distinct {
2641+ distinct = "distinct "
2642+ }
2643+ if !node.Qualifier.IsEmpty() {
2644+ buf.Myprintf("%v.", node.Qualifier)
2645+ }
2646+ // Function names should not be back-quoted even
2647+ // if they match a reserved word. So, print the
2648+ // name as is.
2649+ buf.Myprintf("%s(%s%v)", node.Name.String(), distinct, node.Exprs)
2650+}
2651+
2652+func (node *FuncExpr) walkSubtree(visit Visit) error {
2653+ if node == nil {
2654+ return nil
2655+ }
2656+ return Walk(
2657+ visit,
2658+ node.Qualifier,
2659+ node.Name,
2660+ node.Exprs,
2661+ )
2662+}
2663+
2664+func (node *FuncExpr) replace(from, to Expr) bool {
2665+ for _, sel := range node.Exprs {
2666+ aliased, ok := sel.(*AliasedExpr)
2667+ if !ok {
2668+ continue
2669+ }
2670+ if replaceExprs(from, to, &aliased.Expr) {
2671+ return true
2672+ }
2673+ }
2674+ return false
2675+}
2676+
2677+// Aggregates is a map of all aggregate functions.
2678+var Aggregates = map[string]bool{
2679+ "avg": true,
2680+ "bit_and": true,
2681+ "bit_or": true,
2682+ "bit_xor": true,
2683+ "count": true,
2684+ "group_concat": true,
2685+ "max": true,
2686+ "min": true,
2687+ "std": true,
2688+ "stddev_pop": true,
2689+ "stddev_samp": true,
2690+ "stddev": true,
2691+ "sum": true,
2692+ "var_pop": true,
2693+ "var_samp": true,
2694+ "variance": true,
2695+}
2696+
2697+// IsAggregate returns true if the function is an aggregate.
2698+func (node *FuncExpr) IsAggregate() bool {
2699+ return Aggregates[node.Name.Lowered()]
2700+}
2701+
2702+// GroupConcatExpr represents a call to GROUP_CONCAT
2703+type GroupConcatExpr struct {
2704+ Distinct string
2705+ Exprs SelectExprs
2706+ OrderBy OrderBy
2707+ Separator string
2708+}
2709+
2710+// Format formats the node
2711+func (node *GroupConcatExpr) Format(buf *TrackedBuffer) {
2712+ buf.Myprintf("group_concat(%s%v%v%s)", node.Distinct, node.Exprs, node.OrderBy, node.Separator)
2713+}
2714+
2715+func (node *GroupConcatExpr) walkSubtree(visit Visit) error {
2716+ if node == nil {
2717+ return nil
2718+ }
2719+ return Walk(
2720+ visit,
2721+ node.Exprs,
2722+ node.OrderBy,
2723+ )
2724+}
2725+
2726+func (node *GroupConcatExpr) replace(from, to Expr) bool {
2727+ for _, sel := range node.Exprs {
2728+ aliased, ok := sel.(*AliasedExpr)
2729+ if !ok {
2730+ continue
2731+ }
2732+ if replaceExprs(from, to, &aliased.Expr) {
2733+ return true
2734+ }
2735+ }
2736+ for _, order := range node.OrderBy {
2737+ if replaceExprs(from, to, &order.Expr) {
2738+ return true
2739+ }
2740+ }
2741+ return false
2742+}
2743+
2744+// ValuesFuncExpr represents a function call.
2745+type ValuesFuncExpr struct {
2746+ Name *ColName
2747+}
2748+
2749+// Format formats the node.
2750+func (node *ValuesFuncExpr) Format(buf *TrackedBuffer) {
2751+ buf.Myprintf("values(%v)", node.Name)
2752+}
2753+
2754+func (node *ValuesFuncExpr) walkSubtree(visit Visit) error {
2755+ if node == nil {
2756+ return nil
2757+ }
2758+ return Walk(
2759+ visit,
2760+ node.Name,
2761+ )
2762+}
2763+
2764+func (node *ValuesFuncExpr) replace(from, to Expr) bool {
2765+ return false
2766+}
2767+
2768+// SubstrExpr represents a call to SubstrExpr(column, value_expression) or SubstrExpr(column, value_expression,value_expression)
2769+// also supported syntax SubstrExpr(column from value_expression for value_expression)
2770+type SubstrExpr struct {
2771+ Name *ColName
2772+ From Expr
2773+ To Expr
2774+}
2775+
2776+// Format formats the node.
2777+func (node *SubstrExpr) Format(buf *TrackedBuffer) {
2778+
2779+ if node.To == nil {
2780+ buf.Myprintf("substr(%v, %v)", node.Name, node.From)
2781+ } else {
2782+ buf.Myprintf("substr(%v, %v, %v)", node.Name, node.From, node.To)
2783+ }
2784+}
2785+
2786+func (node *SubstrExpr) replace(from, to Expr) bool {
2787+ return replaceExprs(from, to, &node.From, &node.To)
2788+}
2789+
2790+func (node *SubstrExpr) walkSubtree(visit Visit) error {
2791+ if node == nil {
2792+ return nil
2793+ }
2794+ return Walk(
2795+ visit,
2796+ node.Name,
2797+ node.From,
2798+ node.To,
2799+ )
2800+}
2801+
2802+// ConvertExpr represents a call to CONVERT(expr, type)
2803+// or it's equivalent CAST(expr AS type). Both are rewritten to the former.
2804+type ConvertExpr struct {
2805+ Expr Expr
2806+ Type *ConvertType
2807+}
2808+
2809+// Format formats the node.
2810+func (node *ConvertExpr) Format(buf *TrackedBuffer) {
2811+ buf.Myprintf("convert(%v, %v)", node.Expr, node.Type)
2812+}
2813+
2814+func (node *ConvertExpr) walkSubtree(visit Visit) error {
2815+ if node == nil {
2816+ return nil
2817+ }
2818+ return Walk(
2819+ visit,
2820+ node.Expr,
2821+ node.Type,
2822+ )
2823+}
2824+
2825+func (node *ConvertExpr) replace(from, to Expr) bool {
2826+ return replaceExprs(from, to, &node.Expr)
2827+}
2828+
2829+// ConvertUsingExpr represents a call to CONVERT(expr USING charset).
2830+type ConvertUsingExpr struct {
2831+ Expr Expr
2832+ Type string
2833+}
2834+
2835+// Format formats the node.
2836+func (node *ConvertUsingExpr) Format(buf *TrackedBuffer) {
2837+ buf.Myprintf("convert(%v using %s)", node.Expr, node.Type)
2838+}
2839+
2840+func (node *ConvertUsingExpr) walkSubtree(visit Visit) error {
2841+ if node == nil {
2842+ return nil
2843+ }
2844+ return Walk(
2845+ visit,
2846+ node.Expr,
2847+ )
2848+}
2849+
2850+func (node *ConvertUsingExpr) replace(from, to Expr) bool {
2851+ return replaceExprs(from, to, &node.Expr)
2852+}
2853+
2854+// ConvertType represents the type in call to CONVERT(expr, type)
2855+type ConvertType struct {
2856+ Type string
2857+ Length *SQLVal
2858+ Scale *SQLVal
2859+ Operator string
2860+ Charset string
2861+}
2862+
2863+// this string is "character set" and this comment is required
2864+const (
2865+ CharacterSetStr = " character set"
2866+)
2867+
2868+// Format formats the node.
2869+func (node *ConvertType) Format(buf *TrackedBuffer) {
2870+ buf.Myprintf("%s", node.Type)
2871+ if node.Length != nil {
2872+ buf.Myprintf("(%v", node.Length)
2873+ if node.Scale != nil {
2874+ buf.Myprintf(", %v", node.Scale)
2875+ }
2876+ buf.Myprintf(")")
2877+ }
2878+ if node.Charset != "" {
2879+ buf.Myprintf("%s %s", node.Operator, node.Charset)
2880+ }
2881+}
2882+
2883+func (node *ConvertType) walkSubtree(visit Visit) error {
2884+ return nil
2885+}
2886+
2887+// MatchExpr represents a call to the MATCH function
2888+type MatchExpr struct {
2889+ Columns SelectExprs
2890+ Expr Expr
2891+ Option string
2892+}
2893+
2894+// MatchExpr.Option
2895+const (
2896+ BooleanModeStr = " in boolean mode"
2897+ NaturalLanguageModeStr = " in natural language mode"
2898+ NaturalLanguageModeWithQueryExpansionStr = " in natural language mode with query expansion"
2899+ QueryExpansionStr = " with query expansion"
2900+)
2901+
2902+// Format formats the node
2903+func (node *MatchExpr) Format(buf *TrackedBuffer) {
2904+ buf.Myprintf("match(%v) against (%v%s)", node.Columns, node.Expr, node.Option)
2905+}
2906+
2907+func (node *MatchExpr) walkSubtree(visit Visit) error {
2908+ if node == nil {
2909+ return nil
2910+ }
2911+ return Walk(
2912+ visit,
2913+ node.Columns,
2914+ node.Expr,
2915+ )
2916+}
2917+
2918+func (node *MatchExpr) replace(from, to Expr) bool {
2919+ for _, sel := range node.Columns {
2920+ aliased, ok := sel.(*AliasedExpr)
2921+ if !ok {
2922+ continue
2923+ }
2924+ if replaceExprs(from, to, &aliased.Expr) {
2925+ return true
2926+ }
2927+ }
2928+ return replaceExprs(from, to, &node.Expr)
2929+}
2930+
2931+// CaseExpr represents a CASE expression.
2932+type CaseExpr struct {
2933+ Expr Expr
2934+ Whens []*When
2935+ Else Expr
2936+}
2937+
2938+// Format formats the node.
2939+func (node *CaseExpr) Format(buf *TrackedBuffer) {
2940+ buf.Myprintf("case ")
2941+ if node.Expr != nil {
2942+ buf.Myprintf("%v ", node.Expr)
2943+ }
2944+ for _, when := range node.Whens {
2945+ buf.Myprintf("%v ", when)
2946+ }
2947+ if node.Else != nil {
2948+ buf.Myprintf("else %v ", node.Else)
2949+ }
2950+ buf.Myprintf("end")
2951+}
2952+
2953+func (node *CaseExpr) walkSubtree(visit Visit) error {
2954+ if node == nil {
2955+ return nil
2956+ }
2957+ if err := Walk(visit, node.Expr); err != nil {
2958+ return err
2959+ }
2960+ for _, n := range node.Whens {
2961+ if err := Walk(visit, n); err != nil {
2962+ return err
2963+ }
2964+ }
2965+ return Walk(visit, node.Else)
2966+}
2967+
2968+func (node *CaseExpr) replace(from, to Expr) bool {
2969+ for _, when := range node.Whens {
2970+ if replaceExprs(from, to, &when.Cond, &when.Val) {
2971+ return true
2972+ }
2973+ }
2974+ return replaceExprs(from, to, &node.Expr, &node.Else)
2975+}
2976+
2977+// Default represents a DEFAULT expression.
2978+type Default struct {
2979+ ColName string
2980+}
2981+
2982+// Format formats the node.
2983+func (node *Default) Format(buf *TrackedBuffer) {
2984+ buf.Myprintf("default")
2985+ if node.ColName != "" {
2986+ buf.Myprintf("(%s)", node.ColName)
2987+ }
2988+}
2989+
2990+func (node *Default) walkSubtree(visit Visit) error {
2991+ return nil
2992+}
2993+
2994+func (node *Default) replace(from, to Expr) bool {
2995+ return false
2996+}
2997+
2998+// When represents a WHEN sub-expression.
2999+type When struct {
3000+ Cond Expr
3001+ Val Expr
3002+}
3003+
3004+// Format formats the node.
3005+func (node *When) Format(buf *TrackedBuffer) {
3006+ buf.Myprintf("when %v then %v", node.Cond, node.Val)
3007+}
3008+
3009+func (node *When) walkSubtree(visit Visit) error {
3010+ if node == nil {
3011+ return nil
3012+ }
3013+ return Walk(
3014+ visit,
3015+ node.Cond,
3016+ node.Val,
3017+ )
3018+}
3019+
3020+// GroupBy represents a GROUP BY clause.
3021+type GroupBy []Expr
3022+
3023+// Format formats the node.
3024+func (node GroupBy) Format(buf *TrackedBuffer) {
3025+ prefix := " group by "
3026+ for _, n := range node {
3027+ buf.Myprintf("%s%v", prefix, n)
3028+ prefix = ", "
3029+ }
3030+}
3031+
3032+func (node GroupBy) walkSubtree(visit Visit) error {
3033+ for _, n := range node {
3034+ if err := Walk(visit, n); err != nil {
3035+ return err
3036+ }
3037+ }
3038+ return nil
3039+}
3040+
3041+// OrderBy represents an ORDER By clause.
3042+type OrderBy []*Order
3043+
3044+// Format formats the node.
3045+func (node OrderBy) Format(buf *TrackedBuffer) {
3046+ prefix := " order by "
3047+ for _, n := range node {
3048+ buf.Myprintf("%s%v", prefix, n)
3049+ prefix = ", "
3050+ }
3051+}
3052+
3053+func (node OrderBy) walkSubtree(visit Visit) error {
3054+ for _, n := range node {
3055+ if err := Walk(visit, n); err != nil {
3056+ return err
3057+ }
3058+ }
3059+ return nil
3060+}
3061+
3062+// Order represents an ordering expression.
3063+type Order struct {
3064+ Expr Expr
3065+ Direction string
3066+}
3067+
3068+// Order.Direction
3069+const (
3070+ AscScr = "asc"
3071+ DescScr = "desc"
3072+)
3073+
3074+// Format formats the node.
3075+func (node *Order) Format(buf *TrackedBuffer) {
3076+ if node, ok := node.Expr.(*NullVal); ok {
3077+ buf.Myprintf("%v", node)
3078+ return
3079+ }
3080+ if node, ok := node.Expr.(*FuncExpr); ok {
3081+ if node.Name.Lowered() == "rand" {
3082+ buf.Myprintf("%v", node)
3083+ return
3084+ }
3085+ }
3086+
3087+ buf.Myprintf("%v %s", node.Expr, node.Direction)
3088+}
3089+
3090+func (node *Order) walkSubtree(visit Visit) error {
3091+ if node == nil {
3092+ return nil
3093+ }
3094+ return Walk(
3095+ visit,
3096+ node.Expr,
3097+ )
3098+}
3099+
3100+// Limit represents a LIMIT clause.
3101+type Limit struct {
3102+ Offset, Rowcount Expr
3103+}
3104+
3105+// Format formats the node.
3106+func (node *Limit) Format(buf *TrackedBuffer) {
3107+ if node == nil {
3108+ return
3109+ }
3110+ buf.Myprintf(" limit ")
3111+ if node.Offset != nil {
3112+ buf.Myprintf("%v, ", node.Offset)
3113+ }
3114+ buf.Myprintf("%v", node.Rowcount)
3115+}
3116+
3117+func (node *Limit) walkSubtree(visit Visit) error {
3118+ if node == nil {
3119+ return nil
3120+ }
3121+ return Walk(
3122+ visit,
3123+ node.Offset,
3124+ node.Rowcount,
3125+ )
3126+}
3127+
3128+// Values represents a VALUES clause.
3129+type Values []ValTuple
3130+
3131+// Format formats the node.
3132+func (node Values) Format(buf *TrackedBuffer) {
3133+ prefix := "values "
3134+ for _, n := range node {
3135+ buf.Myprintf("%s%v", prefix, n)
3136+ prefix = ", "
3137+ }
3138+}
3139+
3140+func (node Values) walkSubtree(visit Visit) error {
3141+ for _, n := range node {
3142+ if err := Walk(visit, n); err != nil {
3143+ return err
3144+ }
3145+ }
3146+ return nil
3147+}
3148+
3149+// UpdateExprs represents a list of update expressions.
3150+type UpdateExprs []*UpdateExpr
3151+
3152+// Format formats the node.
3153+func (node UpdateExprs) Format(buf *TrackedBuffer) {
3154+ var prefix string
3155+ for _, n := range node {
3156+ buf.Myprintf("%s%v", prefix, n)
3157+ prefix = ", "
3158+ }
3159+}
3160+
3161+func (node UpdateExprs) walkSubtree(visit Visit) error {
3162+ for _, n := range node {
3163+ if err := Walk(visit, n); err != nil {
3164+ return err
3165+ }
3166+ }
3167+ return nil
3168+}
3169+
3170+// UpdateExpr represents an update expression.
3171+type UpdateExpr struct {
3172+ Name *ColName
3173+ Expr Expr
3174+}
3175+
3176+// Format formats the node.
3177+func (node *UpdateExpr) Format(buf *TrackedBuffer) {
3178+ buf.Myprintf("%v = %v", node.Name, node.Expr)
3179+}
3180+
3181+func (node *UpdateExpr) walkSubtree(visit Visit) error {
3182+ if node == nil {
3183+ return nil
3184+ }
3185+ return Walk(
3186+ visit,
3187+ node.Name,
3188+ node.Expr,
3189+ )
3190+}
3191+
3192+// SetExprs represents a list of set expressions.
3193+type SetExprs []*SetExpr
3194+
3195+// Format formats the node.
3196+func (node SetExprs) Format(buf *TrackedBuffer) {
3197+ var prefix string
3198+ for _, n := range node {
3199+ buf.Myprintf("%s%v", prefix, n)
3200+ prefix = ", "
3201+ }
3202+}
3203+
3204+func (node SetExprs) walkSubtree(visit Visit) error {
3205+ for _, n := range node {
3206+ if err := Walk(visit, n); err != nil {
3207+ return err
3208+ }
3209+ }
3210+ return nil
3211+}
3212+
3213+// SetExpr represents a set expression.
3214+type SetExpr struct {
3215+ Name ColIdent
3216+ Expr Expr
3217+}
3218+
3219+// Format formats the node.
3220+func (node *SetExpr) Format(buf *TrackedBuffer) {
3221+ // We don't have to backtick set variable names.
3222+ if node.Name.EqualString("charset") || node.Name.EqualString("names") {
3223+ buf.Myprintf("%s %v", node.Name.String(), node.Expr)
3224+ } else {
3225+ buf.Myprintf("%s = %v", node.Name.String(), node.Expr)
3226+ }
3227+}
3228+
3229+func (node *SetExpr) walkSubtree(visit Visit) error {
3230+ if node == nil {
3231+ return nil
3232+ }
3233+ return Walk(
3234+ visit,
3235+ node.Name,
3236+ node.Expr,
3237+ )
3238+}
3239+
3240+// OnDup represents an ON DUPLICATE KEY clause.
3241+type OnDup UpdateExprs
3242+
3243+// Format formats the node.
3244+func (node OnDup) Format(buf *TrackedBuffer) {
3245+ if node == nil {
3246+ return
3247+ }
3248+ buf.Myprintf(" on duplicate key update %v", UpdateExprs(node))
3249+}
3250+
3251+func (node OnDup) walkSubtree(visit Visit) error {
3252+ return Walk(visit, UpdateExprs(node))
3253+}
3254+
3255+// ColIdent is a case insensitive SQL identifier. It will be escaped with
3256+// backquotes if necessary.
3257+type ColIdent struct {
3258+ // This artifact prevents this struct from being compared
3259+ // with itself. It consumes no space as long as it's not the
3260+ // last field in the struct.
3261+ _ [0]struct{ _ []byte }
3262+ val, lowered string
3263+}
3264+
3265+// NewColIdent makes a new ColIdent.
3266+func NewColIdent(str string) ColIdent {
3267+ return ColIdent{
3268+ val: str,
3269+ }
3270+}
3271+
3272+// Format formats the node.
3273+func (node ColIdent) Format(buf *TrackedBuffer) {
3274+ formatID(buf, node.val, node.Lowered())
3275+}
3276+
3277+func (node ColIdent) walkSubtree(visit Visit) error {
3278+ return nil
3279+}
3280+
3281+// IsEmpty returns true if the name is empty.
3282+func (node ColIdent) IsEmpty() bool {
3283+ return node.val == ""
3284+}
3285+
3286+// String returns the unescaped column name. It must
3287+// not be used for SQL generation. Use sqlparser.String
3288+// instead. The Stringer conformance is for usage
3289+// in templates.
3290+func (node ColIdent) String() string {
3291+ return node.val
3292+}
3293+
3294+// CompliantName returns a compliant id name
3295+// that can be used for a bind var.
3296+func (node ColIdent) CompliantName() string {
3297+ return compliantName(node.val)
3298+}
3299+
3300+// Lowered returns a lower-cased column name.
3301+// This function should generally be used only for optimizing
3302+// comparisons.
3303+func (node ColIdent) Lowered() string {
3304+ if node.val == "" {
3305+ return ""
3306+ }
3307+ if node.lowered == "" {
3308+ node.lowered = strings.ToLower(node.val)
3309+ }
3310+ return node.lowered
3311+}
3312+
3313+// Equal performs a case-insensitive compare.
3314+func (node ColIdent) Equal(in ColIdent) bool {
3315+ return node.Lowered() == in.Lowered()
3316+}
3317+
3318+// EqualString performs a case-insensitive compare with str.
3319+func (node ColIdent) EqualString(str string) bool {
3320+ return node.Lowered() == strings.ToLower(str)
3321+}
3322+
3323+// MarshalJSON marshals into JSON.
3324+func (node ColIdent) MarshalJSON() ([]byte, error) {
3325+ return json.Marshal(node.val)
3326+}
3327+
3328+// UnmarshalJSON unmarshals from JSON.
3329+func (node *ColIdent) UnmarshalJSON(b []byte) error {
3330+ var result string
3331+ err := json.Unmarshal(b, &result)
3332+ if err != nil {
3333+ return err
3334+ }
3335+ node.val = result
3336+ return nil
3337+}
3338+
3339+// TableIdent is a case sensitive SQL identifier. It will be escaped with
3340+// backquotes if necessary.
3341+type TableIdent struct {
3342+ v string
3343+}
3344+
3345+// NewTableIdent creates a new TableIdent.
3346+func NewTableIdent(str string) TableIdent {
3347+ return TableIdent{v: str}
3348+}
3349+
3350+// Format formats the node.
3351+func (node TableIdent) Format(buf *TrackedBuffer) {
3352+ formatID(buf, node.v, strings.ToLower(node.v))
3353+}
3354+
3355+func (node TableIdent) walkSubtree(visit Visit) error {
3356+ return nil
3357+}
3358+
3359+// IsEmpty returns true if TabIdent is empty.
3360+func (node TableIdent) IsEmpty() bool {
3361+ return node.v == ""
3362+}
3363+
3364+// String returns the unescaped table name. It must
3365+// not be used for SQL generation. Use sqlparser.String
3366+// instead. The Stringer conformance is for usage
3367+// in templates.
3368+func (node TableIdent) String() string {
3369+ return node.v
3370+}
3371+
3372+// CompliantName returns a compliant id name
3373+// that can be used for a bind var.
3374+func (node TableIdent) CompliantName() string {
3375+ return compliantName(node.v)
3376+}
3377+
3378+// MarshalJSON marshals into JSON.
3379+func (node TableIdent) MarshalJSON() ([]byte, error) {
3380+ return json.Marshal(node.v)
3381+}
3382+
3383+// UnmarshalJSON unmarshals from JSON.
3384+func (node *TableIdent) UnmarshalJSON(b []byte) error {
3385+ var result string
3386+ err := json.Unmarshal(b, &result)
3387+ if err != nil {
3388+ return err
3389+ }
3390+ node.v = result
3391+ return nil
3392+}
3393+
3394+// Backtick produces a backticked literal given an input string.
3395+func Backtick(in string) string {
3396+ var buf bytes.Buffer
3397+ buf.WriteByte('`')
3398+ for _, c := range in {
3399+ buf.WriteRune(c)
3400+ if c == '`' {
3401+ buf.WriteByte('`')
3402+ }
3403+ }
3404+ buf.WriteByte('`')
3405+ return buf.String()
3406+}
3407+
3408+func formatID(buf *TrackedBuffer, original, lowered string) {
3409+ isDbSystemVariable := false
3410+ if len(original) > 1 && original[:2] == "@@" {
3411+ isDbSystemVariable = true
3412+ }
3413+
3414+ for i, c := range original {
3415+ if !isLetter(uint16(c)) && (!isDbSystemVariable || !isCarat(uint16(c))) {
3416+ if i == 0 || !isDigit(uint16(c)) {
3417+ goto mustEscape
3418+ }
3419+ }
3420+ }
3421+ if _, ok := keywords[lowered]; ok {
3422+ goto mustEscape
3423+ }
3424+ buf.Myprintf("%s", original)
3425+ return
3426+
3427+mustEscape:
3428+ buf.WriteByte('`')
3429+ for _, c := range original {
3430+ buf.WriteRune(c)
3431+ if c == '`' {
3432+ buf.WriteByte('`')
3433+ }
3434+ }
3435+ buf.WriteByte('`')
3436+}
3437+
3438+func compliantName(in string) string {
3439+ var buf bytes.Buffer
3440+ for i, c := range in {
3441+ if !isLetter(uint16(c)) {
3442+ if i == 0 || !isDigit(uint16(c)) {
3443+ buf.WriteByte('_')
3444+ continue
3445+ }
3446+ }
3447+ buf.WriteRune(c)
3448+ }
3449+ return buf.String()
3450+}
vendor/github.com/xwb1989/sqlparser/comments.go+293−0
@@ -0,0 +1,293 @@
1+/*
2+Copyright 2017 Google Inc.
3+
4+Licensed under the Apache License, Version 2.0 (the "License");
5+you may not use this file except in compliance with the License.
6+You may obtain a copy of the License at
7+
8+ http://www.apache.org/licenses/LICENSE-2.0
9+
10+Unless required by applicable law or agreed to in writing, software
11+distributed under the License is distributed on an "AS IS" BASIS,
12+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+See the License for the specific language governing permissions and
14+limitations under the License.
15+*/
16+
17+package sqlparser
18+
19+import (
20+ "strconv"
21+ "strings"
22+ "unicode"
23+)
24+
25+const (
26+ // DirectiveMultiShardAutocommit is the query comment directive to allow
27+ // single round trip autocommit with a multi-shard statement.
28+ DirectiveMultiShardAutocommit = "MULTI_SHARD_AUTOCOMMIT"
29+ // DirectiveSkipQueryPlanCache skips query plan cache when set.
30+ DirectiveSkipQueryPlanCache = "SKIP_QUERY_PLAN_CACHE"
31+ // DirectiveQueryTimeout sets a query timeout in vtgate. Only supported for SELECTS.
32+ DirectiveQueryTimeout = "QUERY_TIMEOUT_MS"
33+)
34+
35+func isNonSpace(r rune) bool {
36+ return !unicode.IsSpace(r)
37+}
38+
39+// leadingCommentEnd returns the first index after all leading comments, or
40+// 0 if there are no leading comments.
41+func leadingCommentEnd(text string) (end int) {
42+ hasComment := false
43+ pos := 0
44+ for pos < len(text) {
45+ // Eat up any whitespace. Trailing whitespace will be considered part of
46+ // the leading comments.
47+ nextVisibleOffset := strings.IndexFunc(text[pos:], isNonSpace)
48+ if nextVisibleOffset < 0 {
49+ break
50+ }
51+ pos += nextVisibleOffset
52+ remainingText := text[pos:]
53+
54+ // Found visible characters. Look for '/*' at the beginning
55+ // and '*/' somewhere after that.
56+ if len(remainingText) < 4 || remainingText[:2] != "/*" {
57+ break
58+ }
59+ commentLength := 4 + strings.Index(remainingText[2:], "*/")
60+ if commentLength < 4 {
61+ // Missing end comment :/
62+ break
63+ }
64+
65+ hasComment = true
66+ pos += commentLength
67+ }
68+
69+ if hasComment {
70+ return pos
71+ }
72+ return 0
73+}
74+
75+// trailingCommentStart returns the first index of trailing comments.
76+// If there are no trailing comments, returns the length of the input string.
77+func trailingCommentStart(text string) (start int) {
78+ hasComment := false
79+ reducedLen := len(text)
80+ for reducedLen > 0 {
81+ // Eat up any whitespace. Leading whitespace will be considered part of
82+ // the trailing comments.
83+ nextReducedLen := strings.LastIndexFunc(text[:reducedLen], isNonSpace) + 1
84+ if nextReducedLen == 0 {
85+ break
86+ }
87+ reducedLen = nextReducedLen
88+ if reducedLen < 4 || text[reducedLen-2:reducedLen] != "*/" {
89+ break
90+ }
91+
92+ // Find the beginning of the comment
93+ startCommentPos := strings.LastIndex(text[:reducedLen-2], "/*")
94+ if startCommentPos < 0 {
95+ // Badly formatted sql :/
96+ break
97+ }
98+
99+ hasComment = true
100+ reducedLen = startCommentPos
101+ }
102+
103+ if hasComment {
104+ return reducedLen
105+ }
106+ return len(text)
107+}
108+
109+// MarginComments holds the leading and trailing comments that surround a query.
110+type MarginComments struct {
111+ Leading string
112+ Trailing string
113+}
114+
115+// SplitMarginComments pulls out any leading or trailing comments from a raw sql query.
116+// This function also trims leading (if there's a comment) and trailing whitespace.
117+func SplitMarginComments(sql string) (query string, comments MarginComments) {
118+ trailingStart := trailingCommentStart(sql)
119+ leadingEnd := leadingCommentEnd(sql[:trailingStart])
120+ comments = MarginComments{
121+ Leading: strings.TrimLeftFunc(sql[:leadingEnd], unicode.IsSpace),
122+ Trailing: strings.TrimRightFunc(sql[trailingStart:], unicode.IsSpace),
123+ }
124+ return strings.TrimFunc(sql[leadingEnd:trailingStart], unicode.IsSpace), comments
125+}
126+
127+// StripLeadingComments trims the SQL string and removes any leading comments
128+func StripLeadingComments(sql string) string {
129+ sql = strings.TrimFunc(sql, unicode.IsSpace)
130+
131+ for hasCommentPrefix(sql) {
132+ switch sql[0] {
133+ case '/':
134+ // Multi line comment
135+ index := strings.Index(sql, "*/")
136+ if index <= 1 {
137+ return sql
138+ }
139+ // don't strip /*! ... */ or /*!50700 ... */
140+ if len(sql) > 2 && sql[2] == '!' {
141+ return sql
142+ }
143+ sql = sql[index+2:]
144+ case '-':
145+ // Single line comment
146+ index := strings.Index(sql, "\n")
147+ if index == -1 {
148+ return sql
149+ }
150+ sql = sql[index+1:]
151+ }
152+
153+ sql = strings.TrimFunc(sql, unicode.IsSpace)
154+ }
155+
156+ return sql
157+}
158+
159+func hasCommentPrefix(sql string) bool {
160+ return len(sql) > 1 && ((sql[0] == '/' && sql[1] == '*') || (sql[0] == '-' && sql[1] == '-'))
161+}
162+
163+// ExtractMysqlComment extracts the version and SQL from a comment-only query
164+// such as /*!50708 sql here */
165+func ExtractMysqlComment(sql string) (version string, innerSQL string) {
166+ sql = sql[3 : len(sql)-2]
167+
168+ digitCount := 0
169+ endOfVersionIndex := strings.IndexFunc(sql, func(c rune) bool {
170+ digitCount++
171+ return !unicode.IsDigit(c) || digitCount == 6
172+ })
173+ version = sql[0:endOfVersionIndex]
174+ innerSQL = strings.TrimFunc(sql[endOfVersionIndex:], unicode.IsSpace)
175+
176+ return version, innerSQL
177+}
178+
179+const commentDirectivePreamble = "/*vt+"
180+
181+// CommentDirectives is the parsed representation for execution directives
182+// conveyed in query comments
183+type CommentDirectives map[string]interface{}
184+
185+// ExtractCommentDirectives parses the comment list for any execution directives
186+// of the form:
187+//
188+// /*vt+ OPTION_ONE=1 OPTION_TWO OPTION_THREE=abcd */
189+//
190+// It returns the map of the directive values or nil if there aren't any.
191+func ExtractCommentDirectives(comments Comments) CommentDirectives {
192+ if comments == nil {
193+ return nil
194+ }
195+
196+ var vals map[string]interface{}
197+
198+ for _, comment := range comments {
199+ commentStr := string(comment)
200+ if commentStr[0:5] != commentDirectivePreamble {
201+ continue
202+ }
203+
204+ if vals == nil {
205+ vals = make(map[string]interface{})
206+ }
207+
208+ // Split on whitespace and ignore the first and last directive
209+ // since they contain the comment start/end
210+ directives := strings.Fields(commentStr)
211+ for i := 1; i < len(directives)-1; i++ {
212+ directive := directives[i]
213+ sep := strings.IndexByte(directive, '=')
214+
215+ // No value is equivalent to a true boolean
216+ if sep == -1 {
217+ vals[directive] = true
218+ continue
219+ }
220+
221+ strVal := directive[sep+1:]
222+ directive = directive[:sep]
223+
224+ intVal, err := strconv.Atoi(strVal)
225+ if err == nil {
226+ vals[directive] = intVal
227+ continue
228+ }
229+
230+ boolVal, err := strconv.ParseBool(strVal)
231+ if err == nil {
232+ vals[directive] = boolVal
233+ continue
234+ }
235+
236+ vals[directive] = strVal
237+ }
238+ }
239+ return vals
240+}
241+
242+// IsSet checks the directive map for the named directive and returns
243+// true if the directive is set and has a true/false or 0/1 value
244+func (d CommentDirectives) IsSet(key string) bool {
245+ if d == nil {
246+ return false
247+ }
248+
249+ val, ok := d[key]
250+ if !ok {
251+ return false
252+ }
253+
254+ boolVal, ok := val.(bool)
255+ if ok {
256+ return boolVal
257+ }
258+
259+ intVal, ok := val.(int)
260+ if ok {
261+ return intVal == 1
262+ }
263+ return false
264+}
265+
266+// SkipQueryPlanCacheDirective returns true if skip query plan cache directive is set to true in query.
267+func SkipQueryPlanCacheDirective(stmt Statement) bool {
268+ switch stmt := stmt.(type) {
269+ case *Select:
270+ directives := ExtractCommentDirectives(stmt.Comments)
271+ if directives.IsSet(DirectiveSkipQueryPlanCache) {
272+ return true
273+ }
274+ case *Insert:
275+ directives := ExtractCommentDirectives(stmt.Comments)
276+ if directives.IsSet(DirectiveSkipQueryPlanCache) {
277+ return true
278+ }
279+ case *Update:
280+ directives := ExtractCommentDirectives(stmt.Comments)
281+ if directives.IsSet(DirectiveSkipQueryPlanCache) {
282+ return true
283+ }
284+ case *Delete:
285+ directives := ExtractCommentDirectives(stmt.Comments)
286+ if directives.IsSet(DirectiveSkipQueryPlanCache) {
287+ return true
288+ }
289+ default:
290+ return false
291+ }
292+ return false
293+}
vendor/github.com/xwb1989/sqlparser/dependency/bytes2/buffer.go+65−0
@@ -0,0 +1,65 @@
1+/*
2+Copyright 2017 Google Inc.
3+
4+Licensed under the Apache License, Version 2.0 (the "License");
5+you may not use this file except in compliance with the License.
6+You may obtain a copy of the License at
7+
8+ http://www.apache.org/licenses/LICENSE-2.0
9+
10+Unless required by applicable law or agreed to in writing, software
11+distributed under the License is distributed on an "AS IS" BASIS,
12+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+See the License for the specific language governing permissions and
14+limitations under the License.
15+*/
16+
17+package bytes2
18+
19+// Buffer implements a subset of the write portion of
20+// bytes.Buffer, but more efficiently. This is meant to
21+// be used in very high QPS operations, especially for
22+// WriteByte, and without abstracting it as a Writer.
23+// Function signatures contain errors for compatibility,
24+// but they do not return errors.
25+type Buffer struct {
26+ bytes []byte
27+}
28+
29+// NewBuffer is equivalent to bytes.NewBuffer.
30+func NewBuffer(b []byte) *Buffer {
31+ return &Buffer{bytes: b}
32+}
33+
34+// Write is equivalent to bytes.Buffer.Write.
35+func (buf *Buffer) Write(b []byte) (int, error) {
36+ buf.bytes = append(buf.bytes, b...)
37+ return len(b), nil
38+}
39+
40+// WriteString is equivalent to bytes.Buffer.WriteString.
41+func (buf *Buffer) WriteString(s string) (int, error) {
42+ buf.bytes = append(buf.bytes, s...)
43+ return len(s), nil
44+}
45+
46+// WriteByte is equivalent to bytes.Buffer.WriteByte.
47+func (buf *Buffer) WriteByte(b byte) error {
48+ buf.bytes = append(buf.bytes, b)
49+ return nil
50+}
51+
52+// Bytes is equivalent to bytes.Buffer.Bytes.
53+func (buf *Buffer) Bytes() []byte {
54+ return buf.bytes
55+}
56+
57+// Strings is equivalent to bytes.Buffer.Strings.
58+func (buf *Buffer) String() string {
59+ return string(buf.bytes)
60+}
61+
62+// Len is equivalent to bytes.Buffer.Len.
63+func (buf *Buffer) Len() int {
64+ return len(buf.bytes)
65+}
vendor/github.com/xwb1989/sqlparser/dependency/hack/hack.go+79−0
@@ -0,0 +1,79 @@
1+/*
2+Copyright 2017 Google Inc.
3+
4+Licensed under the Apache License, Version 2.0 (the "License");
5+you may not use this file except in compliance with the License.
6+You may obtain a copy of the License at
7+
8+ http://www.apache.org/licenses/LICENSE-2.0
9+
10+Unless required by applicable law or agreed to in writing, software
11+distributed under the License is distributed on an "AS IS" BASIS,
12+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+See the License for the specific language governing permissions and
14+limitations under the License.
15+*/
16+
17+// Package hack gives you some efficient functionality at the cost of
18+// breaking some Go rules.
19+package hack
20+
21+import (
22+ "reflect"
23+ "unsafe"
24+)
25+
26+// StringArena lets you consolidate allocations for a group of strings
27+// that have similar life length
28+type StringArena struct {
29+ buf []byte
30+ str string
31+}
32+
33+// NewStringArena creates an arena of the specified size.
34+func NewStringArena(size int) *StringArena {
35+ sa := &StringArena{buf: make([]byte, 0, size)}
36+ pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&sa.buf))
37+ pstring := (*reflect.StringHeader)(unsafe.Pointer(&sa.str))
38+ pstring.Data = pbytes.Data
39+ pstring.Len = pbytes.Cap
40+ return sa
41+}
42+
43+// NewString copies a byte slice into the arena and returns it as a string.
44+// If the arena is full, it returns a traditional go string.
45+func (sa *StringArena) NewString(b []byte) string {
46+ if len(b) == 0 {
47+ return ""
48+ }
49+ if len(sa.buf)+len(b) > cap(sa.buf) {
50+ return string(b)
51+ }
52+ start := len(sa.buf)
53+ sa.buf = append(sa.buf, b...)
54+ return sa.str[start : start+len(b)]
55+}
56+
57+// SpaceLeft returns the amount of space left in the arena.
58+func (sa *StringArena) SpaceLeft() int {
59+ return cap(sa.buf) - len(sa.buf)
60+}
61+
62+// String force casts a []byte to a string.
63+// USE AT YOUR OWN RISK
64+func String(b []byte) (s string) {
65+ if len(b) == 0 {
66+ return ""
67+ }
68+ pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&b))
69+ pstring := (*reflect.StringHeader)(unsafe.Pointer(&s))
70+ pstring.Data = pbytes.Data
71+ pstring.Len = pbytes.Len
72+ return
73+}
74+
75+// StringPointer returns &s[0], which is not allowed in go
76+func StringPointer(s string) unsafe.Pointer {
77+ pstring := (*reflect.StringHeader)(unsafe.Pointer(&s))
78+ return unsafe.Pointer(pstring.Data)
79+}
vendor/github.com/xwb1989/sqlparser/dependency/querypb/query.pb.go+2734−0
@@ -0,0 +1,2734 @@
1+// Code generated by protoc-gen-go. DO NOT EDIT.
2+// source: query.proto
3+
4+/*
5+Package query is a generated protocol buffer package.
6+
7+It is generated from these files:
8+ query.proto
9+
10+It has these top-level messages:
11+ Target
12+ VTGateCallerID
13+ EventToken
14+ Value
15+ BindVariable
16+ BoundQuery
17+ ExecuteOptions
18+ Field
19+ Row
20+ ResultExtras
21+ QueryResult
22+ StreamEvent
23+ ExecuteRequest
24+ ExecuteResponse
25+ ResultWithError
26+ ExecuteBatchRequest
27+ ExecuteBatchResponse
28+ StreamExecuteRequest
29+ StreamExecuteResponse
30+ BeginRequest
31+ BeginResponse
32+ CommitRequest
33+ CommitResponse
34+ RollbackRequest
35+ RollbackResponse
36+ PrepareRequest
37+ PrepareResponse
38+ CommitPreparedRequest
39+ CommitPreparedResponse
40+ RollbackPreparedRequest
41+ RollbackPreparedResponse
42+ CreateTransactionRequest
43+ CreateTransactionResponse
44+ StartCommitRequest
45+ StartCommitResponse
46+ SetRollbackRequest
47+ SetRollbackResponse
48+ ConcludeTransactionRequest
49+ ConcludeTransactionResponse
50+ ReadTransactionRequest
51+ ReadTransactionResponse
52+ BeginExecuteRequest
53+ BeginExecuteResponse
54+ BeginExecuteBatchRequest
55+ BeginExecuteBatchResponse
56+ MessageStreamRequest
57+ MessageStreamResponse
58+ MessageAckRequest
59+ MessageAckResponse
60+ SplitQueryRequest
61+ QuerySplit
62+ SplitQueryResponse
63+ StreamHealthRequest
64+ RealtimeStats
65+ StreamHealthResponse
66+ UpdateStreamRequest
67+ UpdateStreamResponse
68+ TransactionMetadata
69+*/
70+package querypb
71+
72+import "strconv"
73+
74+// EnumName is a helper function to simplify printing protocol buffer enums
75+// by name. Given an enum map and a value, it returns a useful string.
76+func EnumName(m map[int32]string, v int32) string {
77+ s, ok := m[v]
78+ if ok {
79+ return s
80+ }
81+ return strconv.Itoa(int(v))
82+}
83+
84+// Flags sent from the MySQL C API
85+type MySqlFlag int32
86+
87+const (
88+ MySqlFlag_EMPTY MySqlFlag = 0
89+ MySqlFlag_NOT_NULL_FLAG MySqlFlag = 1
90+ MySqlFlag_PRI_KEY_FLAG MySqlFlag = 2
91+ MySqlFlag_UNIQUE_KEY_FLAG MySqlFlag = 4
92+ MySqlFlag_MULTIPLE_KEY_FLAG MySqlFlag = 8
93+ MySqlFlag_BLOB_FLAG MySqlFlag = 16
94+ MySqlFlag_UNSIGNED_FLAG MySqlFlag = 32
95+ MySqlFlag_ZEROFILL_FLAG MySqlFlag = 64
96+ MySqlFlag_BINARY_FLAG MySqlFlag = 128
97+ MySqlFlag_ENUM_FLAG MySqlFlag = 256
98+ MySqlFlag_AUTO_INCREMENT_FLAG MySqlFlag = 512
99+ MySqlFlag_TIMESTAMP_FLAG MySqlFlag = 1024
100+ MySqlFlag_SET_FLAG MySqlFlag = 2048
101+ MySqlFlag_NO_DEFAULT_VALUE_FLAG MySqlFlag = 4096
102+ MySqlFlag_ON_UPDATE_NOW_FLAG MySqlFlag = 8192
103+ MySqlFlag_NUM_FLAG MySqlFlag = 32768
104+ MySqlFlag_PART_KEY_FLAG MySqlFlag = 16384
105+ MySqlFlag_GROUP_FLAG MySqlFlag = 32768
106+ MySqlFlag_UNIQUE_FLAG MySqlFlag = 65536
107+ MySqlFlag_BINCMP_FLAG MySqlFlag = 131072
108+)
109+
110+var MySqlFlag_name = map[int32]string{
111+ 0: "EMPTY",
112+ 1: "NOT_NULL_FLAG",
113+ 2: "PRI_KEY_FLAG",
114+ 4: "UNIQUE_KEY_FLAG",
115+ 8: "MULTIPLE_KEY_FLAG",
116+ 16: "BLOB_FLAG",
117+ 32: "UNSIGNED_FLAG",
118+ 64: "ZEROFILL_FLAG",
119+ 128: "BINARY_FLAG",
120+ 256: "ENUM_FLAG",
121+ 512: "AUTO_INCREMENT_FLAG",
122+ 1024: "TIMESTAMP_FLAG",
123+ 2048: "SET_FLAG",
124+ 4096: "NO_DEFAULT_VALUE_FLAG",
125+ 8192: "ON_UPDATE_NOW_FLAG",
126+ 32768: "NUM_FLAG",
127+ 16384: "PART_KEY_FLAG",
128+ // Duplicate value: 32768: "GROUP_FLAG",
129+ 65536: "UNIQUE_FLAG",
130+ 131072: "BINCMP_FLAG",
131+}
132+var MySqlFlag_value = map[string]int32{
133+ "EMPTY": 0,
134+ "NOT_NULL_FLAG": 1,
135+ "PRI_KEY_FLAG": 2,
136+ "UNIQUE_KEY_FLAG": 4,
137+ "MULTIPLE_KEY_FLAG": 8,
138+ "BLOB_FLAG": 16,
139+ "UNSIGNED_FLAG": 32,
140+ "ZEROFILL_FLAG": 64,
141+ "BINARY_FLAG": 128,
142+ "ENUM_FLAG": 256,
143+ "AUTO_INCREMENT_FLAG": 512,
144+ "TIMESTAMP_FLAG": 1024,
145+ "SET_FLAG": 2048,
146+ "NO_DEFAULT_VALUE_FLAG": 4096,
147+ "ON_UPDATE_NOW_FLAG": 8192,
148+ "NUM_FLAG": 32768,
149+ "PART_KEY_FLAG": 16384,
150+ "GROUP_FLAG": 32768,
151+ "UNIQUE_FLAG": 65536,
152+ "BINCMP_FLAG": 131072,
153+}
154+
155+func (x MySqlFlag) String() string {
156+ return EnumName(MySqlFlag_name, int32(x))
157+}
158+
159+// Flag allows us to qualify types by their common properties.
160+type Flag int32
161+
162+const (
163+ Flag_NONE Flag = 0
164+ Flag_ISINTEGRAL Flag = 256
165+ Flag_ISUNSIGNED Flag = 512
166+ Flag_ISFLOAT Flag = 1024
167+ Flag_ISQUOTED Flag = 2048
168+ Flag_ISTEXT Flag = 4096
169+ Flag_ISBINARY Flag = 8192
170+)
171+
172+var Flag_name = map[int32]string{
173+ 0: "NONE",
174+ 256: "ISINTEGRAL",
175+ 512: "ISUNSIGNED",
176+ 1024: "ISFLOAT",
177+ 2048: "ISQUOTED",
178+ 4096: "ISTEXT",
179+ 8192: "ISBINARY",
180+}
181+var Flag_value = map[string]int32{
182+ "NONE": 0,
183+ "ISINTEGRAL": 256,
184+ "ISUNSIGNED": 512,
185+ "ISFLOAT": 1024,
186+ "ISQUOTED": 2048,
187+ "ISTEXT": 4096,
188+ "ISBINARY": 8192,
189+}
190+
191+func (x Flag) String() string {
192+ return EnumName(Flag_name, int32(x))
193+}
194+
195+// Type defines the various supported data types in bind vars
196+// and query results.
197+type Type int32
198+
199+const (
200+ // NULL_TYPE specifies a NULL type.
201+ Type_NULL_TYPE Type = 0
202+ // INT8 specifies a TINYINT type.
203+ // Properties: 1, IsNumber.
204+ Type_INT8 Type = 257
205+ // UINT8 specifies a TINYINT UNSIGNED type.
206+ // Properties: 2, IsNumber, IsUnsigned.
207+ Type_UINT8 Type = 770
208+ // INT16 specifies a SMALLINT type.
209+ // Properties: 3, IsNumber.
210+ Type_INT16 Type = 259
211+ // UINT16 specifies a SMALLINT UNSIGNED type.
212+ // Properties: 4, IsNumber, IsUnsigned.
213+ Type_UINT16 Type = 772
214+ // INT24 specifies a MEDIUMINT type.
215+ // Properties: 5, IsNumber.
216+ Type_INT24 Type = 261
217+ // UINT24 specifies a MEDIUMINT UNSIGNED type.
218+ // Properties: 6, IsNumber, IsUnsigned.
219+ Type_UINT24 Type = 774
220+ // INT32 specifies a INTEGER type.
221+ // Properties: 7, IsNumber.
222+ Type_INT32 Type = 263
223+ // UINT32 specifies a INTEGER UNSIGNED type.
224+ // Properties: 8, IsNumber, IsUnsigned.
225+ Type_UINT32 Type = 776
226+ // INT64 specifies a BIGINT type.
227+ // Properties: 9, IsNumber.
228+ Type_INT64 Type = 265
229+ // UINT64 specifies a BIGINT UNSIGNED type.
230+ // Properties: 10, IsNumber, IsUnsigned.
231+ Type_UINT64 Type = 778
232+ // FLOAT32 specifies a FLOAT type.
233+ // Properties: 11, IsFloat.
234+ Type_FLOAT32 Type = 1035
235+ // FLOAT64 specifies a DOUBLE or REAL type.
236+ // Properties: 12, IsFloat.
237+ Type_FLOAT64 Type = 1036
238+ // TIMESTAMP specifies a TIMESTAMP type.
239+ // Properties: 13, IsQuoted.
240+ Type_TIMESTAMP Type = 2061
241+ // DATE specifies a DATE type.
242+ // Properties: 14, IsQuoted.
243+ Type_DATE Type = 2062
244+ // TIME specifies a TIME type.
245+ // Properties: 15, IsQuoted.
246+ Type_TIME Type = 2063
247+ // DATETIME specifies a DATETIME type.
248+ // Properties: 16, IsQuoted.
249+ Type_DATETIME Type = 2064
250+ // YEAR specifies a YEAR type.
251+ // Properties: 17, IsNumber, IsUnsigned.
252+ Type_YEAR Type = 785
253+ // DECIMAL specifies a DECIMAL or NUMERIC type.
254+ // Properties: 18, None.
255+ Type_DECIMAL Type = 18
256+ // TEXT specifies a TEXT type.
257+ // Properties: 19, IsQuoted, IsText.
258+ Type_TEXT Type = 6163
259+ // BLOB specifies a BLOB type.
260+ // Properties: 20, IsQuoted, IsBinary.
261+ Type_BLOB Type = 10260
262+ // VARCHAR specifies a VARCHAR type.
263+ // Properties: 21, IsQuoted, IsText.
264+ Type_VARCHAR Type = 6165
265+ // VARBINARY specifies a VARBINARY type.
266+ // Properties: 22, IsQuoted, IsBinary.
267+ Type_VARBINARY Type = 10262
268+ // CHAR specifies a CHAR type.
269+ // Properties: 23, IsQuoted, IsText.
270+ Type_CHAR Type = 6167
271+ // BINARY specifies a BINARY type.
272+ // Properties: 24, IsQuoted, IsBinary.
273+ Type_BINARY Type = 10264
274+ // BIT specifies a BIT type.
275+ // Properties: 25, IsQuoted.
276+ Type_BIT Type = 2073
277+ // ENUM specifies an ENUM type.
278+ // Properties: 26, IsQuoted.
279+ Type_ENUM Type = 2074
280+ // SET specifies a SET type.
281+ // Properties: 27, IsQuoted.
282+ Type_SET Type = 2075
283+ // TUPLE specifies a a tuple. This cannot
284+ // be returned in a QueryResult, but it can
285+ // be sent as a bind var.
286+ // Properties: 28, None.
287+ Type_TUPLE Type = 28
288+ // GEOMETRY specifies a GEOMETRY type.
289+ // Properties: 29, IsQuoted.
290+ Type_GEOMETRY Type = 2077
291+ // JSON specifies a JSON type.
292+ // Properties: 30, IsQuoted.
293+ Type_JSON Type = 2078
294+ // EXPRESSION specifies a SQL expression.
295+ // This type is for internal use only.
296+ // Properties: 31, None.
297+ Type_EXPRESSION Type = 31
298+)
299+
300+var Type_name = map[int32]string{
301+ 0: "NULL_TYPE",
302+ 257: "INT8",
303+ 770: "UINT8",
304+ 259: "INT16",
305+ 772: "UINT16",
306+ 261: "INT24",
307+ 774: "UINT24",
308+ 263: "INT32",
309+ 776: "UINT32",
310+ 265: "INT64",
311+ 778: "UINT64",
312+ 1035: "FLOAT32",
313+ 1036: "FLOAT64",
314+ 2061: "TIMESTAMP",
315+ 2062: "DATE",
316+ 2063: "TIME",
317+ 2064: "DATETIME",
318+ 785: "YEAR",
319+ 18: "DECIMAL",
320+ 6163: "TEXT",
321+ 10260: "BLOB",
322+ 6165: "VARCHAR",
323+ 10262: "VARBINARY",
324+ 6167: "CHAR",
325+ 10264: "BINARY",
326+ 2073: "BIT",
327+ 2074: "ENUM",
328+ 2075: "SET",
329+ 28: "TUPLE",
330+ 2077: "GEOMETRY",
331+ 2078: "JSON",
332+ 31: "EXPRESSION",
333+}
334+var Type_value = map[string]int32{
335+ "NULL_TYPE": 0,
336+ "INT8": 257,
337+ "UINT8": 770,
338+ "INT16": 259,
339+ "UINT16": 772,
340+ "INT24": 261,
341+ "UINT24": 774,
342+ "INT32": 263,
343+ "UINT32": 776,
344+ "INT64": 265,
345+ "UINT64": 778,
346+ "FLOAT32": 1035,
347+ "FLOAT64": 1036,
348+ "TIMESTAMP": 2061,
349+ "DATE": 2062,
350+ "TIME": 2063,
351+ "DATETIME": 2064,
352+ "YEAR": 785,
353+ "DECIMAL": 18,
354+ "TEXT": 6163,
355+ "BLOB": 10260,
356+ "VARCHAR": 6165,
357+ "VARBINARY": 10262,
358+ "CHAR": 6167,
359+ "BINARY": 10264,
360+ "BIT": 2073,
361+ "ENUM": 2074,
362+ "SET": 2075,
363+ "TUPLE": 28,
364+ "GEOMETRY": 2077,
365+ "JSON": 2078,
366+ "EXPRESSION": 31,
367+}
368+
369+func (x Type) String() string {
370+ return EnumName(Type_name, int32(x))
371+}
372+
373+// TransactionState represents the state of a distributed transaction.
374+type TransactionState int32
375+
376+const (
377+ TransactionState_UNKNOWN TransactionState = 0
378+ TransactionState_PREPARE TransactionState = 1
379+ TransactionState_COMMIT TransactionState = 2
380+ TransactionState_ROLLBACK TransactionState = 3
381+)
382+
383+var TransactionState_name = map[int32]string{
384+ 0: "UNKNOWN",
385+ 1: "PREPARE",
386+ 2: "COMMIT",
387+ 3: "ROLLBACK",
388+}
389+var TransactionState_value = map[string]int32{
390+ "UNKNOWN": 0,
391+ "PREPARE": 1,
392+ "COMMIT": 2,
393+ "ROLLBACK": 3,
394+}
395+
396+func (x TransactionState) String() string {
397+ return EnumName(TransactionState_name, int32(x))
398+}
399+
400+type ExecuteOptions_IncludedFields int32
401+
402+const (
403+ ExecuteOptions_TYPE_AND_NAME ExecuteOptions_IncludedFields = 0
404+ ExecuteOptions_TYPE_ONLY ExecuteOptions_IncludedFields = 1
405+ ExecuteOptions_ALL ExecuteOptions_IncludedFields = 2
406+)
407+
408+var ExecuteOptions_IncludedFields_name = map[int32]string{
409+ 0: "TYPE_AND_NAME",
410+ 1: "TYPE_ONLY",
411+ 2: "ALL",
412+}
413+var ExecuteOptions_IncludedFields_value = map[string]int32{
414+ "TYPE_AND_NAME": 0,
415+ "TYPE_ONLY": 1,
416+ "ALL": 2,
417+}
418+
419+func (x ExecuteOptions_IncludedFields) String() string {
420+ return EnumName(ExecuteOptions_IncludedFields_name, int32(x))
421+}
422+
423+type ExecuteOptions_Workload int32
424+
425+const (
426+ ExecuteOptions_UNSPECIFIED ExecuteOptions_Workload = 0
427+ ExecuteOptions_OLTP ExecuteOptions_Workload = 1
428+ ExecuteOptions_OLAP ExecuteOptions_Workload = 2
429+ ExecuteOptions_DBA ExecuteOptions_Workload = 3
430+)
431+
432+var ExecuteOptions_Workload_name = map[int32]string{
433+ 0: "UNSPECIFIED",
434+ 1: "OLTP",
435+ 2: "OLAP",
436+ 3: "DBA",
437+}
438+var ExecuteOptions_Workload_value = map[string]int32{
439+ "UNSPECIFIED": 0,
440+ "OLTP": 1,
441+ "OLAP": 2,
442+ "DBA": 3,
443+}
444+
445+func (x ExecuteOptions_Workload) String() string {
446+ return EnumName(ExecuteOptions_Workload_name, int32(x))
447+}
448+
449+type ExecuteOptions_TransactionIsolation int32
450+
451+const (
452+ ExecuteOptions_DEFAULT ExecuteOptions_TransactionIsolation = 0
453+ ExecuteOptions_REPEATABLE_READ ExecuteOptions_TransactionIsolation = 1
454+ ExecuteOptions_READ_COMMITTED ExecuteOptions_TransactionIsolation = 2
455+ ExecuteOptions_READ_UNCOMMITTED ExecuteOptions_TransactionIsolation = 3
456+ ExecuteOptions_SERIALIZABLE ExecuteOptions_TransactionIsolation = 4
457+)
458+
459+var ExecuteOptions_TransactionIsolation_name = map[int32]string{
460+ 0: "DEFAULT",
461+ 1: "REPEATABLE_READ",
462+ 2: "READ_COMMITTED",
463+ 3: "READ_UNCOMMITTED",
464+ 4: "SERIALIZABLE",
465+}
466+var ExecuteOptions_TransactionIsolation_value = map[string]int32{
467+ "DEFAULT": 0,
468+ "REPEATABLE_READ": 1,
469+ "READ_COMMITTED": 2,
470+ "READ_UNCOMMITTED": 3,
471+ "SERIALIZABLE": 4,
472+}
473+
474+func (x ExecuteOptions_TransactionIsolation) String() string {
475+ return EnumName(ExecuteOptions_TransactionIsolation_name, int32(x))
476+}
477+
478+// The category of one statement.
479+type StreamEvent_Statement_Category int32
480+
481+const (
482+ StreamEvent_Statement_Error StreamEvent_Statement_Category = 0
483+ StreamEvent_Statement_DML StreamEvent_Statement_Category = 1
484+ StreamEvent_Statement_DDL StreamEvent_Statement_Category = 2
485+)
486+
487+var StreamEvent_Statement_Category_name = map[int32]string{
488+ 0: "Error",
489+ 1: "DML",
490+ 2: "DDL",
491+}
492+var StreamEvent_Statement_Category_value = map[string]int32{
493+ "Error": 0,
494+ "DML": 1,
495+ "DDL": 2,
496+}
497+
498+func (x StreamEvent_Statement_Category) String() string {
499+ return EnumName(StreamEvent_Statement_Category_name, int32(x))
500+}
501+
502+type SplitQueryRequest_Algorithm int32
503+
504+const (
505+ SplitQueryRequest_EQUAL_SPLITS SplitQueryRequest_Algorithm = 0
506+ SplitQueryRequest_FULL_SCAN SplitQueryRequest_Algorithm = 1
507+)
508+
509+var SplitQueryRequest_Algorithm_name = map[int32]string{
510+ 0: "EQUAL_SPLITS",
511+ 1: "FULL_SCAN",
512+}
513+var SplitQueryRequest_Algorithm_value = map[string]int32{
514+ "EQUAL_SPLITS": 0,
515+ "FULL_SCAN": 1,
516+}
517+
518+func (x SplitQueryRequest_Algorithm) String() string {
519+ return EnumName(SplitQueryRequest_Algorithm_name, int32(x))
520+}
521+
522+/*
523+// Target describes what the client expects the tablet is.
524+// If the tablet does not match, an error is returned.
525+type Target struct {
526+ Keyspace string `protobuf:"bytes,1,opt,name=keyspace" json:"keyspace,omitempty"`
527+ Shard string `protobuf:"bytes,2,opt,name=shard" json:"shard,omitempty"`
528+ TabletType topodata.TabletType `protobuf:"varint,3,opt,name=tablet_type,json=tabletType,enum=topodata.TabletType" json:"tablet_type,omitempty"`
529+}
530+
531+func (m *Target) Reset() { *m = Target{} }
532+func (m *Target) String() string { return "TODO" }
533+
534+
535+
536+func (m *Target) GetKeyspace() string {
537+ if m != nil {
538+ return m.Keyspace
539+ }
540+ return ""
541+}
542+
543+func (m *Target) GetShard() string {
544+ if m != nil {
545+ return m.Shard
546+ }
547+ return ""
548+}
549+
550+func (m *Target) GetTabletType() topodata.TabletType {
551+ if m != nil {
552+ return m.TabletType
553+ }
554+ return topodata.TabletType_UNKNOWN
555+}
556+
557+
558+// VTGateCallerID is sent by VTGate to VTTablet to describe the
559+// caller. If possible, this information is secure. For instance,
560+// if using unique certificates that guarantee that VTGate->VTTablet
561+// traffic cannot be spoofed, then VTTablet can trust this information,
562+// and VTTablet will use it for tablet ACLs, for instance.
563+// Because of this security guarantee, this is different than the CallerID
564+// structure, which is not secure at all, because it is provided
565+// by the Vitess client.
566+type VTGateCallerID struct {
567+ Username string `protobuf:"bytes,1,opt,name=username" json:"username,omitempty"`
568+ Groups []string `protobuf:"bytes,2,rep,name=groups" json:"groups,omitempty"`
569+}
570+
571+func (m *VTGateCallerID) Reset() { *m = VTGateCallerID{} }
572+func (m *VTGateCallerID) String() string { return "TODO" }
573+
574+
575+
576+func (m *VTGateCallerID) GetUsername() string {
577+ if m != nil {
578+ return m.Username
579+ }
580+ return ""
581+}
582+
583+func (m *VTGateCallerID) GetGroups() []string {
584+ if m != nil {
585+ return m.Groups
586+ }
587+ return nil
588+}
589+
590+// EventToken is a structure that describes a point in time in a
591+// replication stream on one shard. The most recent known replication
592+// position can be retrieved from vttablet when executing a query. It
593+// is also sent with the replication streams from the binlog service.
594+type EventToken struct {
595+ // timestamp is the MySQL timestamp of the statements. Seconds since Epoch.
596+ Timestamp int64 `protobuf:"varint,1,opt,name=timestamp" json:"timestamp,omitempty"`
597+ // The shard name that applied the statements. Note this is not set when
598+ // streaming from a vttablet. It is only used on the client -> vtgate link.
599+ Shard string `protobuf:"bytes,2,opt,name=shard" json:"shard,omitempty"`
600+ // The position on the replication stream after this statement was applied.
601+ // It is not the transaction ID / GTID, but the position / GTIDSet.
602+ Position string `protobuf:"bytes,3,opt,name=position" json:"position,omitempty"`
603+}
604+
605+func (m *EventToken) Reset() { *m = EventToken{} }
606+func (m *EventToken) String() string { return "TODO" }
607+
608+
609+
610+func (m *EventToken) GetTimestamp() int64 {
611+ if m != nil {
612+ return m.Timestamp
613+ }
614+ return 0
615+}
616+
617+func (m *EventToken) GetShard() string {
618+ if m != nil {
619+ return m.Shard
620+ }
621+ return ""
622+}
623+
624+func (m *EventToken) GetPosition() string {
625+ if m != nil {
626+ return m.Position
627+ }
628+ return ""
629+}
630+*/
631+
632+// Value represents a typed value.
633+type Value struct {
634+ Type Type `protobuf:"varint,1,opt,name=type,enum=query.Type" json:"type,omitempty"`
635+ Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
636+}
637+
638+func (m *Value) Reset() { *m = Value{} }
639+func (m *Value) String() string { return "TODO" }
640+
641+func (m *Value) GetType() Type {
642+ if m != nil {
643+ return m.Type
644+ }
645+ return Type_NULL_TYPE
646+}
647+
648+func (m *Value) GetValue() []byte {
649+ if m != nil {
650+ return m.Value
651+ }
652+ return nil
653+}
654+
655+// BindVariable represents a single bind variable in a Query.
656+type BindVariable struct {
657+ Type Type `protobuf:"varint,1,opt,name=type,enum=query.Type" json:"type,omitempty"`
658+ Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
659+ // values are set if type is TUPLE.
660+ Values []*Value `protobuf:"bytes,3,rep,name=values" json:"values,omitempty"`
661+}
662+
663+func (m *BindVariable) Reset() { *m = BindVariable{} }
664+func (m *BindVariable) String() string { return "TODO" }
665+
666+func (m *BindVariable) GetType() Type {
667+ if m != nil {
668+ return m.Type
669+ }
670+ return Type_NULL_TYPE
671+}
672+
673+func (m *BindVariable) GetValue() []byte {
674+ if m != nil {
675+ return m.Value
676+ }
677+ return nil
678+}
679+
680+func (m *BindVariable) GetValues() []*Value {
681+ if m != nil {
682+ return m.Values
683+ }
684+ return nil
685+}
686+
687+// BoundQuery is a query with its bind variables
688+type BoundQuery struct {
689+ // sql is the SQL query to execute
690+ Sql string `protobuf:"bytes,1,opt,name=sql" json:"sql,omitempty"`
691+ // bind_variables is a map of all bind variables to expand in the query.
692+ // nil values are not allowed. Use NULL_TYPE to express a NULL value.
693+ BindVariables map[string]*BindVariable `protobuf:"bytes,2,rep,name=bind_variables,json=bindVariables" json:"bind_variables,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
694+}
695+
696+func (m *BoundQuery) Reset() { *m = BoundQuery{} }
697+func (m *BoundQuery) String() string { return "TODO" }
698+
699+func (m *BoundQuery) GetSql() string {
700+ if m != nil {
701+ return m.Sql
702+ }
703+ return ""
704+}
705+
706+func (m *BoundQuery) GetBindVariables() map[string]*BindVariable {
707+ if m != nil {
708+ return m.BindVariables
709+ }
710+ return nil
711+}
712+
713+/*
714+// ExecuteOptions is passed around for all Execute calls.
715+type ExecuteOptions struct {
716+ // If set, we will try to include an EventToken with the responses.
717+ IncludeEventToken bool `protobuf:"varint,2,opt,name=include_event_token,json=includeEventToken" json:"include_event_token,omitempty"`
718+ // If set, the fresher field may be set as a result comparison to this token.
719+ // This is a shortcut so the application doesn't need to care about
720+ // comparing EventTokens.
721+ CompareEventToken *EventToken `protobuf:"bytes,3,opt,name=compare_event_token,json=compareEventToken" json:"compare_event_token,omitempty"`
722+ // Controls what fields are returned in Field message responses from mysql, i.e.
723+ // field name, table name, etc. This is an optimization for high-QPS queries where
724+ // the client knows what it's getting
725+ IncludedFields ExecuteOptions_IncludedFields `protobuf:"varint,4,opt,name=included_fields,json=includedFields,enum=query.ExecuteOptions_IncludedFields" json:"included_fields,omitempty"`
726+ // client_rows_found specifies if rows_affected should return
727+ // rows found instead of rows affected. Behavior is defined
728+ // by MySQL's CLIENT_FOUND_ROWS flag.
729+ ClientFoundRows bool `protobuf:"varint,5,opt,name=client_found_rows,json=clientFoundRows" json:"client_found_rows,omitempty"`
730+ // workload specifies the type of workload:
731+ // OLTP: DMLs allowed, results have row count limit, and
732+ // query timeouts are shorter.
733+ // OLAP: DMLS not allowed, no limit on row count, timeouts
734+ // can be as high as desired.
735+ // DBA: no limit on rowcount or timeout, all queries allowed
736+ // but intended for long DMLs and DDLs.
737+ Workload ExecuteOptions_Workload `protobuf:"varint,6,opt,name=workload,enum=query.ExecuteOptions_Workload" json:"workload,omitempty"`
738+ // sql_select_limit sets an implicit limit on all select statements. Since
739+ // vitess also sets a rowcount limit on queries, the smallest value wins.
740+ SqlSelectLimit int64 `protobuf:"varint,8,opt,name=sql_select_limit,json=sqlSelectLimit" json:"sql_select_limit,omitempty"`
741+ TransactionIsolation ExecuteOptions_TransactionIsolation `protobuf:"varint,9,opt,name=transaction_isolation,json=transactionIsolation,enum=query.ExecuteOptions_TransactionIsolation" json:"transaction_isolation,omitempty"`
742+}
743+
744+func (m *ExecuteOptions) Reset() { *m = ExecuteOptions{} }
745+func (m *ExecuteOptions) String() string { return "TODO" }
746+
747+
748+
749+func (m *ExecuteOptions) GetIncludeEventToken() bool {
750+ if m != nil {
751+ return m.IncludeEventToken
752+ }
753+ return false
754+}
755+
756+func (m *ExecuteOptions) GetCompareEventToken() *EventToken {
757+ if m != nil {
758+ return m.CompareEventToken
759+ }
760+ return nil
761+}
762+
763+func (m *ExecuteOptions) GetIncludedFields() ExecuteOptions_IncludedFields {
764+ if m != nil {
765+ return m.IncludedFields
766+ }
767+ return ExecuteOptions_TYPE_AND_NAME
768+}
769+
770+func (m *ExecuteOptions) GetClientFoundRows() bool {
771+ if m != nil {
772+ return m.ClientFoundRows
773+ }
774+ return false
775+}
776+
777+func (m *ExecuteOptions) GetWorkload() ExecuteOptions_Workload {
778+ if m != nil {
779+ return m.Workload
780+ }
781+ return ExecuteOptions_UNSPECIFIED
782+}
783+
784+func (m *ExecuteOptions) GetSqlSelectLimit() int64 {
785+ if m != nil {
786+ return m.SqlSelectLimit
787+ }
788+ return 0
789+}
790+
791+func (m *ExecuteOptions) GetTransactionIsolation() ExecuteOptions_TransactionIsolation {
792+ if m != nil {
793+ return m.TransactionIsolation
794+ }
795+ return ExecuteOptions_DEFAULT
796+}
797+
798+// Field describes a single column returned by a query
799+type Field struct {
800+ // name of the field as returned by mysql C API
801+ Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
802+ // vitess-defined type. Conversion function is in sqltypes package.
803+ Type Type `protobuf:"varint,2,opt,name=type,enum=query.Type" json:"type,omitempty"`
804+ // Remaining fields from mysql C API.
805+ // These fields are only populated when ExecuteOptions.included_fields
806+ // is set to IncludedFields.ALL.
807+ Table string `protobuf:"bytes,3,opt,name=table" json:"table,omitempty"`
808+ OrgTable string `protobuf:"bytes,4,opt,name=org_table,json=orgTable" json:"org_table,omitempty"`
809+ Database string `protobuf:"bytes,5,opt,name=database" json:"database,omitempty"`
810+ OrgName string `protobuf:"bytes,6,opt,name=org_name,json=orgName" json:"org_name,omitempty"`
811+ // column_length is really a uint32. All 32 bits can be used.
812+ ColumnLength uint32 `protobuf:"varint,7,opt,name=column_length,json=columnLength" json:"column_length,omitempty"`
813+ // charset is actually a uint16. Only the lower 16 bits are used.
814+ Charset uint32 `protobuf:"varint,8,opt,name=charset" json:"charset,omitempty"`
815+ // decimals is actualy a uint8. Only the lower 8 bits are used.
816+ Decimals uint32 `protobuf:"varint,9,opt,name=decimals" json:"decimals,omitempty"`
817+ // flags is actually a uint16. Only the lower 16 bits are used.
818+ Flags uint32 `protobuf:"varint,10,opt,name=flags" json:"flags,omitempty"`
819+}
820+
821+func (m *Field) Reset() { *m = Field{} }
822+func (m *Field) String() string { return "TODO" }
823+
824+
825+
826+func (m *Field) GetName() string {
827+ if m != nil {
828+ return m.Name
829+ }
830+ return ""
831+}
832+
833+func (m *Field) GetType() Type {
834+ if m != nil {
835+ return m.Type
836+ }
837+ return Type_NULL_TYPE
838+}
839+
840+func (m *Field) GetTable() string {
841+ if m != nil {
842+ return m.Table
843+ }
844+ return ""
845+}
846+
847+func (m *Field) GetOrgTable() string {
848+ if m != nil {
849+ return m.OrgTable
850+ }
851+ return ""
852+}
853+
854+func (m *Field) GetDatabase() string {
855+ if m != nil {
856+ return m.Database
857+ }
858+ return ""
859+}
860+
861+func (m *Field) GetOrgName() string {
862+ if m != nil {
863+ return m.OrgName
864+ }
865+ return ""
866+}
867+
868+func (m *Field) GetColumnLength() uint32 {
869+ if m != nil {
870+ return m.ColumnLength
871+ }
872+ return 0
873+}
874+
875+func (m *Field) GetCharset() uint32 {
876+ if m != nil {
877+ return m.Charset
878+ }
879+ return 0
880+}
881+
882+func (m *Field) GetDecimals() uint32 {
883+ if m != nil {
884+ return m.Decimals
885+ }
886+ return 0
887+}
888+
889+func (m *Field) GetFlags() uint32 {
890+ if m != nil {
891+ return m.Flags
892+ }
893+ return 0
894+}
895+
896+// Row is a database row.
897+type Row struct {
898+ // lengths contains the length of each value in values.
899+ // A length of -1 means that the field is NULL. While
900+ // reading values, you have to accummulate the length
901+ // to know the offset where the next value begins in values.
902+ Lengths []int64 `protobuf:"zigzag64,1,rep,packed,name=lengths" json:"lengths,omitempty"`
903+ // values contains a concatenation of all values in the row.
904+ Values []byte `protobuf:"bytes,2,opt,name=values,proto3" json:"values,omitempty"`
905+}
906+
907+func (m *Row) Reset() { *m = Row{} }
908+func (m *Row) String() string { return "TODO" }
909+
910+
911+
912+func (m *Row) GetLengths() []int64 {
913+ if m != nil {
914+ return m.Lengths
915+ }
916+ return nil
917+}
918+
919+func (m *Row) GetValues() []byte {
920+ if m != nil {
921+ return m.Values
922+ }
923+ return nil
924+}
925+
926+// ResultExtras contains optional out-of-band information. Usually the
927+// extras are requested by adding ExecuteOptions flags.
928+type ResultExtras struct {
929+ // event_token is populated if the include_event_token flag is set
930+ // in ExecuteOptions.
931+ EventToken *EventToken `protobuf:"bytes,1,opt,name=event_token,json=eventToken" json:"event_token,omitempty"`
932+ // If set, it means the data returned with this result is fresher
933+ // than the compare_token passed in the ExecuteOptions.
934+ Fresher bool `protobuf:"varint,2,opt,name=fresher" json:"fresher,omitempty"`
935+}
936+
937+func (m *ResultExtras) Reset() { *m = ResultExtras{} }
938+func (m *ResultExtras) String() string { return "TODO" }
939+
940+
941+
942+func (m *ResultExtras) GetEventToken() *EventToken {
943+ if m != nil {
944+ return m.EventToken
945+ }
946+ return nil
947+}
948+
949+func (m *ResultExtras) GetFresher() bool {
950+ if m != nil {
951+ return m.Fresher
952+ }
953+ return false
954+}
955+
956+// QueryResult is returned by Execute and ExecuteStream.
957+//
958+// As returned by Execute, len(fields) is always equal to len(row)
959+// (for each row in rows).
960+//
961+// As returned by StreamExecute, the first QueryResult has the fields
962+// set, and subsequent QueryResult have rows set. And as Execute,
963+// len(QueryResult[0].fields) is always equal to len(row) (for each
964+// row in rows for each QueryResult in QueryResult[1:]).
965+type QueryResult struct {
966+ Fields []*Field `protobuf:"bytes,1,rep,name=fields" json:"fields,omitempty"`
967+ RowsAffected uint64 `protobuf:"varint,2,opt,name=rows_affected,json=rowsAffected" json:"rows_affected,omitempty"`
968+ InsertId uint64 `protobuf:"varint,3,opt,name=insert_id,json=insertId" json:"insert_id,omitempty"`
969+ Rows []*Row `protobuf:"bytes,4,rep,name=rows" json:"rows,omitempty"`
970+ Extras *ResultExtras `protobuf:"bytes,5,opt,name=extras" json:"extras,omitempty"`
971+}
972+
973+func (m *QueryResult) Reset() { *m = QueryResult{} }
974+func (m *QueryResult) String() string { return "TODO" }
975+
976+
977+
978+func (m *QueryResult) GetFields() []*Field {
979+ if m != nil {
980+ return m.Fields
981+ }
982+ return nil
983+}
984+
985+func (m *QueryResult) GetRowsAffected() uint64 {
986+ if m != nil {
987+ return m.RowsAffected
988+ }
989+ return 0
990+}
991+
992+func (m *QueryResult) GetInsertId() uint64 {
993+ if m != nil {
994+ return m.InsertId
995+ }
996+ return 0
997+}
998+
999+func (m *QueryResult) GetRows() []*Row {
1000+ if m != nil {
1001+ return m.Rows
1002+ }
1003+ return nil
1004+}
1005+
1006+func (m *QueryResult) GetExtras() *ResultExtras {
1007+ if m != nil {
1008+ return m.Extras
1009+ }
1010+ return nil
1011+}
1012+
1013+// StreamEvent describes a set of transformations that happened as a
1014+// single transactional unit on a server. It is streamed back by the
1015+// Update Stream calls.
1016+type StreamEvent struct {
1017+ // The statements in this transaction.
1018+ Statements []*StreamEvent_Statement `protobuf:"bytes,1,rep,name=statements" json:"statements,omitempty"`
1019+ // The Event Token for this event.
1020+ EventToken *EventToken `protobuf:"bytes,2,opt,name=event_token,json=eventToken" json:"event_token,omitempty"`
1021+}
1022+
1023+func (m *StreamEvent) Reset() { *m = StreamEvent{} }
1024+func (m *StreamEvent) String() string { return "TODO" }
1025+
1026+
1027+
1028+func (m *StreamEvent) GetStatements() []*StreamEvent_Statement {
1029+ if m != nil {
1030+ return m.Statements
1031+ }
1032+ return nil
1033+}
1034+
1035+func (m *StreamEvent) GetEventToken() *EventToken {
1036+ if m != nil {
1037+ return m.EventToken
1038+ }
1039+ return nil
1040+}
1041+
1042+// One individual Statement in a transaction.
1043+type StreamEvent_Statement struct {
1044+ Category StreamEvent_Statement_Category `protobuf:"varint,1,opt,name=category,enum=query.StreamEvent_Statement_Category" json:"category,omitempty"`
1045+ // table_name, primary_key_fields and primary_key_values are set for DML.
1046+ TableName string `protobuf:"bytes,2,opt,name=table_name,json=tableName" json:"table_name,omitempty"`
1047+ PrimaryKeyFields []*Field `protobuf:"bytes,3,rep,name=primary_key_fields,json=primaryKeyFields" json:"primary_key_fields,omitempty"`
1048+ PrimaryKeyValues []*Row `protobuf:"bytes,4,rep,name=primary_key_values,json=primaryKeyValues" json:"primary_key_values,omitempty"`
1049+ // sql is set for all queries.
1050+ // FIXME(alainjobart) we may not need it for DMLs.
1051+ Sql []byte `protobuf:"bytes,5,opt,name=sql,proto3" json:"sql,omitempty"`
1052+}
1053+
1054+func (m *StreamEvent_Statement) Reset() { *m = StreamEvent_Statement{} }
1055+func (m *StreamEvent_Statement) String() string { return "TODO" }
1056+
1057+
1058+
1059+func (m *StreamEvent_Statement) GetCategory() StreamEvent_Statement_Category {
1060+ if m != nil {
1061+ return m.Category
1062+ }
1063+ return StreamEvent_Statement_Error
1064+}
1065+
1066+func (m *StreamEvent_Statement) GetTableName() string {
1067+ if m != nil {
1068+ return m.TableName
1069+ }
1070+ return ""
1071+}
1072+
1073+func (m *StreamEvent_Statement) GetPrimaryKeyFields() []*Field {
1074+ if m != nil {
1075+ return m.PrimaryKeyFields
1076+ }
1077+ return nil
1078+}
1079+
1080+func (m *StreamEvent_Statement) GetPrimaryKeyValues() []*Row {
1081+ if m != nil {
1082+ return m.PrimaryKeyValues
1083+ }
1084+ return nil
1085+}
1086+
1087+func (m *StreamEvent_Statement) GetSql() []byte {
1088+ if m != nil {
1089+ return m.Sql
1090+ }
1091+ return nil
1092+}
1093+
1094+
1095+// ExecuteRequest is the payload to Execute
1096+type ExecuteRequest struct {
1097+ EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
1098+ ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
1099+ Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
1100+ Query *BoundQuery `protobuf:"bytes,4,opt,name=query" json:"query,omitempty"`
1101+ TransactionId int64 `protobuf:"varint,5,opt,name=transaction_id,json=transactionId" json:"transaction_id,omitempty"`
1102+ Options *ExecuteOptions `protobuf:"bytes,6,opt,name=options" json:"options,omitempty"`
1103+}
1104+
1105+func (m *ExecuteRequest) Reset() { *m = ExecuteRequest{} }
1106+func (m *ExecuteRequest) String() string { return "TODO" }
1107+
1108+
1109+
1110+func (m *ExecuteRequest) GetEffectiveCallerId() *vtrpc.CallerID {
1111+ if m != nil {
1112+ return m.EffectiveCallerId
1113+ }
1114+ return nil
1115+}
1116+
1117+func (m *ExecuteRequest) GetImmediateCallerId() *VTGateCallerID {
1118+ if m != nil {
1119+ return m.ImmediateCallerId
1120+ }
1121+ return nil
1122+}
1123+
1124+func (m *ExecuteRequest) GetTarget() *Target {
1125+ if m != nil {
1126+ return m.Target
1127+ }
1128+ return nil
1129+}
1130+
1131+func (m *ExecuteRequest) GetQuery() *BoundQuery {
1132+ if m != nil {
1133+ return m.Query
1134+ }
1135+ return nil
1136+}
1137+
1138+func (m *ExecuteRequest) GetTransactionId() int64 {
1139+ if m != nil {
1140+ return m.TransactionId
1141+ }
1142+ return 0
1143+}
1144+
1145+func (m *ExecuteRequest) GetOptions() *ExecuteOptions {
1146+ if m != nil {
1147+ return m.Options
1148+ }
1149+ return nil
1150+}
1151+
1152+// ExecuteResponse is the returned value from Execute
1153+type ExecuteResponse struct {
1154+ Result *QueryResult `protobuf:"bytes,1,opt,name=result" json:"result,omitempty"`
1155+}
1156+
1157+func (m *ExecuteResponse) Reset() { *m = ExecuteResponse{} }
1158+func (m *ExecuteResponse) String() string { return "TODO" }
1159+
1160+
1161+
1162+func (m *ExecuteResponse) GetResult() *QueryResult {
1163+ if m != nil {
1164+ return m.Result
1165+ }
1166+ return nil
1167+}
1168+
1169+// ResultWithError represents a query response
1170+// in the form of result or error but not both.
1171+// TODO: To be used in ExecuteBatchResponse and BeginExecuteBatchResponse.
1172+type ResultWithError struct {
1173+ // error contains an query level error, only set if result is unset.
1174+ Error *vtrpc.RPCError `protobuf:"bytes,1,opt,name=error" json:"error,omitempty"`
1175+ // result contains the query result, only set if error is unset.
1176+ Result *QueryResult `protobuf:"bytes,2,opt,name=result" json:"result,omitempty"`
1177+}
1178+
1179+func (m *ResultWithError) Reset() { *m = ResultWithError{} }
1180+func (m *ResultWithError) String() string { return "TODO" }
1181+
1182+
1183+
1184+func (m *ResultWithError) GetError() *vtrpc.RPCError {
1185+ if m != nil {
1186+ return m.Error
1187+ }
1188+ return nil
1189+}
1190+
1191+func (m *ResultWithError) GetResult() *QueryResult {
1192+ if m != nil {
1193+ return m.Result
1194+ }
1195+ return nil
1196+}
1197+
1198+// ExecuteBatchRequest is the payload to ExecuteBatch
1199+type ExecuteBatchRequest struct {
1200+ EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
1201+ ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
1202+ Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
1203+ Queries []*BoundQuery `protobuf:"bytes,4,rep,name=queries" json:"queries,omitempty"`
1204+ AsTransaction bool `protobuf:"varint,5,opt,name=as_transaction,json=asTransaction" json:"as_transaction,omitempty"`
1205+ TransactionId int64 `protobuf:"varint,6,opt,name=transaction_id,json=transactionId" json:"transaction_id,omitempty"`
1206+ Options *ExecuteOptions `protobuf:"bytes,7,opt,name=options" json:"options,omitempty"`
1207+}
1208+
1209+func (m *ExecuteBatchRequest) Reset() { *m = ExecuteBatchRequest{} }
1210+func (m *ExecuteBatchRequest) String() string { return "TODO" }
1211+
1212+
1213+
1214+func (m *ExecuteBatchRequest) GetEffectiveCallerId() *vtrpc.CallerID {
1215+ if m != nil {
1216+ return m.EffectiveCallerId
1217+ }
1218+ return nil
1219+}
1220+
1221+func (m *ExecuteBatchRequest) GetImmediateCallerId() *VTGateCallerID {
1222+ if m != nil {
1223+ return m.ImmediateCallerId
1224+ }
1225+ return nil
1226+}
1227+
1228+func (m *ExecuteBatchRequest) GetTarget() *Target {
1229+ if m != nil {
1230+ return m.Target
1231+ }
1232+ return nil
1233+}
1234+
1235+func (m *ExecuteBatchRequest) GetQueries() []*BoundQuery {
1236+ if m != nil {
1237+ return m.Queries
1238+ }
1239+ return nil
1240+}
1241+
1242+func (m *ExecuteBatchRequest) GetAsTransaction() bool {
1243+ if m != nil {
1244+ return m.AsTransaction
1245+ }
1246+ return false
1247+}
1248+
1249+func (m *ExecuteBatchRequest) GetTransactionId() int64 {
1250+ if m != nil {
1251+ return m.TransactionId
1252+ }
1253+ return 0
1254+}
1255+
1256+func (m *ExecuteBatchRequest) GetOptions() *ExecuteOptions {
1257+ if m != nil {
1258+ return m.Options
1259+ }
1260+ return nil
1261+}
1262+
1263+// ExecuteBatchResponse is the returned value from ExecuteBatch
1264+type ExecuteBatchResponse struct {
1265+ Results []*QueryResult `protobuf:"bytes,1,rep,name=results" json:"results,omitempty"`
1266+}
1267+
1268+func (m *ExecuteBatchResponse) Reset() { *m = ExecuteBatchResponse{} }
1269+func (m *ExecuteBatchResponse) String() string { return "TODO" }
1270+
1271+
1272+
1273+func (m *ExecuteBatchResponse) GetResults() []*QueryResult {
1274+ if m != nil {
1275+ return m.Results
1276+ }
1277+ return nil
1278+}
1279+
1280+// StreamExecuteRequest is the payload to StreamExecute
1281+type StreamExecuteRequest struct {
1282+ EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
1283+ ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
1284+ Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
1285+ Query *BoundQuery `protobuf:"bytes,4,opt,name=query" json:"query,omitempty"`
1286+ Options *ExecuteOptions `protobuf:"bytes,5,opt,name=options" json:"options,omitempty"`
1287+}
1288+
1289+func (m *StreamExecuteRequest) Reset() { *m = StreamExecuteRequest{} }
1290+func (m *StreamExecuteRequest) String() string { return "TODO" }
1291+
1292+
1293+
1294+func (m *StreamExecuteRequest) GetEffectiveCallerId() *vtrpc.CallerID {
1295+ if m != nil {
1296+ return m.EffectiveCallerId
1297+ }
1298+ return nil
1299+}
1300+
1301+func (m *StreamExecuteRequest) GetImmediateCallerId() *VTGateCallerID {
1302+ if m != nil {
1303+ return m.ImmediateCallerId
1304+ }
1305+ return nil
1306+}
1307+
1308+func (m *StreamExecuteRequest) GetTarget() *Target {
1309+ if m != nil {
1310+ return m.Target
1311+ }
1312+ return nil
1313+}
1314+
1315+func (m *StreamExecuteRequest) GetQuery() *BoundQuery {
1316+ if m != nil {
1317+ return m.Query
1318+ }
1319+ return nil
1320+}
1321+
1322+func (m *StreamExecuteRequest) GetOptions() *ExecuteOptions {
1323+ if m != nil {
1324+ return m.Options
1325+ }
1326+ return nil
1327+}
1328+
1329+// StreamExecuteResponse is the returned value from StreamExecute
1330+type StreamExecuteResponse struct {
1331+ Result *QueryResult `protobuf:"bytes,1,opt,name=result" json:"result,omitempty"`
1332+}
1333+
1334+func (m *StreamExecuteResponse) Reset() { *m = StreamExecuteResponse{} }
1335+func (m *StreamExecuteResponse) String() string { return "TODO" }
1336+
1337+
1338+
1339+func (m *StreamExecuteResponse) GetResult() *QueryResult {
1340+ if m != nil {
1341+ return m.Result
1342+ }
1343+ return nil
1344+}
1345+
1346+// BeginRequest is the payload to Begin
1347+type BeginRequest struct {
1348+ EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
1349+ ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
1350+ Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
1351+ Options *ExecuteOptions `protobuf:"bytes,4,opt,name=options" json:"options,omitempty"`
1352+}
1353+
1354+func (m *BeginRequest) Reset() { *m = BeginRequest{} }
1355+func (m *BeginRequest) String() string { return "TODO" }
1356+
1357+
1358+
1359+func (m *BeginRequest) GetEffectiveCallerId() *vtrpc.CallerID {
1360+ if m != nil {
1361+ return m.EffectiveCallerId
1362+ }
1363+ return nil
1364+}
1365+
1366+func (m *BeginRequest) GetImmediateCallerId() *VTGateCallerID {
1367+ if m != nil {
1368+ return m.ImmediateCallerId
1369+ }
1370+ return nil
1371+}
1372+
1373+func (m *BeginRequest) GetTarget() *Target {
1374+ if m != nil {
1375+ return m.Target
1376+ }
1377+ return nil
1378+}
1379+
1380+func (m *BeginRequest) GetOptions() *ExecuteOptions {
1381+ if m != nil {
1382+ return m.Options
1383+ }
1384+ return nil
1385+}
1386+
1387+// BeginResponse is the returned value from Begin
1388+type BeginResponse struct {
1389+ TransactionId int64 `protobuf:"varint,1,opt,name=transaction_id,json=transactionId" json:"transaction_id,omitempty"`
1390+}
1391+
1392+func (m *BeginResponse) Reset() { *m = BeginResponse{} }
1393+func (m *BeginResponse) String() string { return "TODO" }
1394+
1395+
1396+
1397+func (m *BeginResponse) GetTransactionId() int64 {
1398+ if m != nil {
1399+ return m.TransactionId
1400+ }
1401+ return 0
1402+}
1403+
1404+// CommitRequest is the payload to Commit
1405+type CommitRequest struct {
1406+ EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
1407+ ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
1408+ Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
1409+ TransactionId int64 `protobuf:"varint,4,opt,name=transaction_id,json=transactionId" json:"transaction_id,omitempty"`
1410+}
1411+
1412+func (m *CommitRequest) Reset() { *m = CommitRequest{} }
1413+func (m *CommitRequest) String() string { return "TODO" }
1414+
1415+
1416+
1417+func (m *CommitRequest) GetEffectiveCallerId() *vtrpc.CallerID {
1418+ if m != nil {
1419+ return m.EffectiveCallerId
1420+ }
1421+ return nil
1422+}
1423+
1424+func (m *CommitRequest) GetImmediateCallerId() *VTGateCallerID {
1425+ if m != nil {
1426+ return m.ImmediateCallerId
1427+ }
1428+ return nil
1429+}
1430+
1431+func (m *CommitRequest) GetTarget() *Target {
1432+ if m != nil {
1433+ return m.Target
1434+ }
1435+ return nil
1436+}
1437+
1438+func (m *CommitRequest) GetTransactionId() int64 {
1439+ if m != nil {
1440+ return m.TransactionId
1441+ }
1442+ return 0
1443+}
1444+
1445+// CommitResponse is the returned value from Commit
1446+type CommitResponse struct {
1447+}
1448+
1449+func (m *CommitResponse) Reset() { *m = CommitResponse{} }
1450+func (m *CommitResponse) String() string { return "TODO" }
1451+
1452+
1453+
1454+// RollbackRequest is the payload to Rollback
1455+type RollbackRequest struct {
1456+ EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
1457+ ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
1458+ Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
1459+ TransactionId int64 `protobuf:"varint,4,opt,name=transaction_id,json=transactionId" json:"transaction_id,omitempty"`
1460+}
1461+
1462+func (m *RollbackRequest) Reset() { *m = RollbackRequest{} }
1463+func (m *RollbackRequest) String() string { return "TODO" }
1464+
1465+
1466+
1467+func (m *RollbackRequest) GetEffectiveCallerId() *vtrpc.CallerID {
1468+ if m != nil {
1469+ return m.EffectiveCallerId
1470+ }
1471+ return nil
1472+}
1473+
1474+func (m *RollbackRequest) GetImmediateCallerId() *VTGateCallerID {
1475+ if m != nil {
1476+ return m.ImmediateCallerId
1477+ }
1478+ return nil
1479+}
1480+
1481+func (m *RollbackRequest) GetTarget() *Target {
1482+ if m != nil {
1483+ return m.Target
1484+ }
1485+ return nil
1486+}
1487+
1488+func (m *RollbackRequest) GetTransactionId() int64 {
1489+ if m != nil {
1490+ return m.TransactionId
1491+ }
1492+ return 0
1493+}
1494+
1495+// RollbackResponse is the returned value from Rollback
1496+type RollbackResponse struct {
1497+}
1498+
1499+func (m *RollbackResponse) Reset() { *m = RollbackResponse{} }
1500+func (m *RollbackResponse) String() string { return "TODO" }
1501+
1502+
1503+
1504+// PrepareRequest is the payload to Prepare
1505+type PrepareRequest struct {
1506+ EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
1507+ ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
1508+ Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
1509+ TransactionId int64 `protobuf:"varint,4,opt,name=transaction_id,json=transactionId" json:"transaction_id,omitempty"`
1510+ Dtid string `protobuf:"bytes,5,opt,name=dtid" json:"dtid,omitempty"`
1511+}
1512+
1513+func (m *PrepareRequest) Reset() { *m = PrepareRequest{} }
1514+func (m *PrepareRequest) String() string { return "TODO" }
1515+
1516+
1517+
1518+func (m *PrepareRequest) GetEffectiveCallerId() *vtrpc.CallerID {
1519+ if m != nil {
1520+ return m.EffectiveCallerId
1521+ }
1522+ return nil
1523+}
1524+
1525+func (m *PrepareRequest) GetImmediateCallerId() *VTGateCallerID {
1526+ if m != nil {
1527+ return m.ImmediateCallerId
1528+ }
1529+ return nil
1530+}
1531+
1532+func (m *PrepareRequest) GetTarget() *Target {
1533+ if m != nil {
1534+ return m.Target
1535+ }
1536+ return nil
1537+}
1538+
1539+func (m *PrepareRequest) GetTransactionId() int64 {
1540+ if m != nil {
1541+ return m.TransactionId
1542+ }
1543+ return 0
1544+}
1545+
1546+func (m *PrepareRequest) GetDtid() string {
1547+ if m != nil {
1548+ return m.Dtid
1549+ }
1550+ return ""
1551+}
1552+
1553+// PrepareResponse is the returned value from Prepare
1554+type PrepareResponse struct {
1555+}
1556+
1557+func (m *PrepareResponse) Reset() { *m = PrepareResponse{} }
1558+func (m *PrepareResponse) String() string { return "TODO" }
1559+
1560+
1561+
1562+// CommitPreparedRequest is the payload to CommitPrepared
1563+type CommitPreparedRequest struct {
1564+ EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
1565+ ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
1566+ Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
1567+ Dtid string `protobuf:"bytes,4,opt,name=dtid" json:"dtid,omitempty"`
1568+}
1569+
1570+func (m *CommitPreparedRequest) Reset() { *m = CommitPreparedRequest{} }
1571+func (m *CommitPreparedRequest) String() string { return "TODO" }
1572+
1573+
1574+
1575+func (m *CommitPreparedRequest) GetEffectiveCallerId() *vtrpc.CallerID {
1576+ if m != nil {
1577+ return m.EffectiveCallerId
1578+ }
1579+ return nil
1580+}
1581+
1582+func (m *CommitPreparedRequest) GetImmediateCallerId() *VTGateCallerID {
1583+ if m != nil {
1584+ return m.ImmediateCallerId
1585+ }
1586+ return nil
1587+}
1588+
1589+func (m *CommitPreparedRequest) GetTarget() *Target {
1590+ if m != nil {
1591+ return m.Target
1592+ }
1593+ return nil
1594+}
1595+
1596+func (m *CommitPreparedRequest) GetDtid() string {
1597+ if m != nil {
1598+ return m.Dtid
1599+ }
1600+ return ""
1601+}
1602+
1603+// CommitPreparedResponse is the returned value from CommitPrepared
1604+type CommitPreparedResponse struct {
1605+}
1606+
1607+func (m *CommitPreparedResponse) Reset() { *m = CommitPreparedResponse{} }
1608+func (m *CommitPreparedResponse) String() string { return "TODO" }
1609+
1610+
1611+
1612+// RollbackPreparedRequest is the payload to RollbackPrepared
1613+type RollbackPreparedRequest struct {
1614+ EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
1615+ ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
1616+ Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
1617+ TransactionId int64 `protobuf:"varint,4,opt,name=transaction_id,json=transactionId" json:"transaction_id,omitempty"`
1618+ Dtid string `protobuf:"bytes,5,opt,name=dtid" json:"dtid,omitempty"`
1619+}
1620+
1621+func (m *RollbackPreparedRequest) Reset() { *m = RollbackPreparedRequest{} }
1622+func (m *RollbackPreparedRequest) String() string { return "TODO" }
1623+
1624+
1625+
1626+func (m *RollbackPreparedRequest) GetEffectiveCallerId() *vtrpc.CallerID {
1627+ if m != nil {
1628+ return m.EffectiveCallerId
1629+ }
1630+ return nil
1631+}
1632+
1633+func (m *RollbackPreparedRequest) GetImmediateCallerId() *VTGateCallerID {
1634+ if m != nil {
1635+ return m.ImmediateCallerId
1636+ }
1637+ return nil
1638+}
1639+
1640+func (m *RollbackPreparedRequest) GetTarget() *Target {
1641+ if m != nil {
1642+ return m.Target
1643+ }
1644+ return nil
1645+}
1646+
1647+func (m *RollbackPreparedRequest) GetTransactionId() int64 {
1648+ if m != nil {
1649+ return m.TransactionId
1650+ }
1651+ return 0
1652+}
1653+
1654+func (m *RollbackPreparedRequest) GetDtid() string {
1655+ if m != nil {
1656+ return m.Dtid
1657+ }
1658+ return ""
1659+}
1660+
1661+// RollbackPreparedResponse is the returned value from RollbackPrepared
1662+type RollbackPreparedResponse struct {
1663+}
1664+
1665+func (m *RollbackPreparedResponse) Reset() { *m = RollbackPreparedResponse{} }
1666+func (m *RollbackPreparedResponse) String() string { return "TODO" }
1667+
1668+
1669+
1670+// CreateTransactionRequest is the payload to CreateTransaction
1671+type CreateTransactionRequest struct {
1672+ EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
1673+ ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
1674+ Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
1675+ Dtid string `protobuf:"bytes,4,opt,name=dtid" json:"dtid,omitempty"`
1676+ Participants []*Target `protobuf:"bytes,5,rep,name=participants" json:"participants,omitempty"`
1677+}
1678+
1679+func (m *CreateTransactionRequest) Reset() { *m = CreateTransactionRequest{} }
1680+func (m *CreateTransactionRequest) String() string { return "TODO" }
1681+
1682+
1683+
1684+func (m *CreateTransactionRequest) GetEffectiveCallerId() *vtrpc.CallerID {
1685+ if m != nil {
1686+ return m.EffectiveCallerId
1687+ }
1688+ return nil
1689+}
1690+
1691+func (m *CreateTransactionRequest) GetImmediateCallerId() *VTGateCallerID {
1692+ if m != nil {
1693+ return m.ImmediateCallerId
1694+ }
1695+ return nil
1696+}
1697+
1698+func (m *CreateTransactionRequest) GetTarget() *Target {
1699+ if m != nil {
1700+ return m.Target
1701+ }
1702+ return nil
1703+}
1704+
1705+func (m *CreateTransactionRequest) GetDtid() string {
1706+ if m != nil {
1707+ return m.Dtid
1708+ }
1709+ return ""
1710+}
1711+
1712+func (m *CreateTransactionRequest) GetParticipants() []*Target {
1713+ if m != nil {
1714+ return m.Participants
1715+ }
1716+ return nil
1717+}
1718+
1719+// CreateTransactionResponse is the returned value from CreateTransaction
1720+type CreateTransactionResponse struct {
1721+}
1722+
1723+func (m *CreateTransactionResponse) Reset() { *m = CreateTransactionResponse{} }
1724+func (m *CreateTransactionResponse) String() string { return "TODO" }
1725+
1726+
1727+
1728+// StartCommitRequest is the payload to StartCommit
1729+type StartCommitRequest struct {
1730+ EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
1731+ ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
1732+ Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
1733+ TransactionId int64 `protobuf:"varint,4,opt,name=transaction_id,json=transactionId" json:"transaction_id,omitempty"`
1734+ Dtid string `protobuf:"bytes,5,opt,name=dtid" json:"dtid,omitempty"`
1735+}
1736+
1737+func (m *StartCommitRequest) Reset() { *m = StartCommitRequest{} }
1738+func (m *StartCommitRequest) String() string { return "TODO" }
1739+
1740+
1741+
1742+func (m *StartCommitRequest) GetEffectiveCallerId() *vtrpc.CallerID {
1743+ if m != nil {
1744+ return m.EffectiveCallerId
1745+ }
1746+ return nil
1747+}
1748+
1749+func (m *StartCommitRequest) GetImmediateCallerId() *VTGateCallerID {
1750+ if m != nil {
1751+ return m.ImmediateCallerId
1752+ }
1753+ return nil
1754+}
1755+
1756+func (m *StartCommitRequest) GetTarget() *Target {
1757+ if m != nil {
1758+ return m.Target
1759+ }
1760+ return nil
1761+}
1762+
1763+func (m *StartCommitRequest) GetTransactionId() int64 {
1764+ if m != nil {
1765+ return m.TransactionId
1766+ }
1767+ return 0
1768+}
1769+
1770+func (m *StartCommitRequest) GetDtid() string {
1771+ if m != nil {
1772+ return m.Dtid
1773+ }
1774+ return ""
1775+}
1776+
1777+// StartCommitResponse is the returned value from StartCommit
1778+type StartCommitResponse struct {
1779+}
1780+
1781+func (m *StartCommitResponse) Reset() { *m = StartCommitResponse{} }
1782+func (m *StartCommitResponse) String() string { return "TODO" }
1783+
1784+
1785+
1786+// SetRollbackRequest is the payload to SetRollback
1787+type SetRollbackRequest struct {
1788+ EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
1789+ ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
1790+ Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
1791+ TransactionId int64 `protobuf:"varint,4,opt,name=transaction_id,json=transactionId" json:"transaction_id,omitempty"`
1792+ Dtid string `protobuf:"bytes,5,opt,name=dtid" json:"dtid,omitempty"`
1793+}
1794+
1795+func (m *SetRollbackRequest) Reset() { *m = SetRollbackRequest{} }
1796+func (m *SetRollbackRequest) String() string { return "TODO" }
1797+
1798+
1799+
1800+func (m *SetRollbackRequest) GetEffectiveCallerId() *vtrpc.CallerID {
1801+ if m != nil {
1802+ return m.EffectiveCallerId
1803+ }
1804+ return nil
1805+}
1806+
1807+func (m *SetRollbackRequest) GetImmediateCallerId() *VTGateCallerID {
1808+ if m != nil {
1809+ return m.ImmediateCallerId
1810+ }
1811+ return nil
1812+}
1813+
1814+func (m *SetRollbackRequest) GetTarget() *Target {
1815+ if m != nil {
1816+ return m.Target
1817+ }
1818+ return nil
1819+}
1820+
1821+func (m *SetRollbackRequest) GetTransactionId() int64 {
1822+ if m != nil {
1823+ return m.TransactionId
1824+ }
1825+ return 0
1826+}
1827+
1828+func (m *SetRollbackRequest) GetDtid() string {
1829+ if m != nil {
1830+ return m.Dtid
1831+ }
1832+ return ""
1833+}
1834+
1835+// SetRollbackResponse is the returned value from SetRollback
1836+type SetRollbackResponse struct {
1837+}
1838+
1839+func (m *SetRollbackResponse) Reset() { *m = SetRollbackResponse{} }
1840+func (m *SetRollbackResponse) String() string { return "TODO" }
1841+
1842+
1843+
1844+// ConcludeTransactionRequest is the payload to ConcludeTransaction
1845+type ConcludeTransactionRequest struct {
1846+ EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
1847+ ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
1848+ Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
1849+ Dtid string `protobuf:"bytes,4,opt,name=dtid" json:"dtid,omitempty"`
1850+}
1851+
1852+func (m *ConcludeTransactionRequest) Reset() { *m = ConcludeTransactionRequest{} }
1853+func (m *ConcludeTransactionRequest) String() string { return "TODO" }
1854+
1855+
1856+
1857+func (m *ConcludeTransactionRequest) GetEffectiveCallerId() *vtrpc.CallerID {
1858+ if m != nil {
1859+ return m.EffectiveCallerId
1860+ }
1861+ return nil
1862+}
1863+
1864+func (m *ConcludeTransactionRequest) GetImmediateCallerId() *VTGateCallerID {
1865+ if m != nil {
1866+ return m.ImmediateCallerId
1867+ }
1868+ return nil
1869+}
1870+
1871+func (m *ConcludeTransactionRequest) GetTarget() *Target {
1872+ if m != nil {
1873+ return m.Target
1874+ }
1875+ return nil
1876+}
1877+
1878+func (m *ConcludeTransactionRequest) GetDtid() string {
1879+ if m != nil {
1880+ return m.Dtid
1881+ }
1882+ return ""
1883+}
1884+
1885+// ConcludeTransactionResponse is the returned value from ConcludeTransaction
1886+type ConcludeTransactionResponse struct {
1887+}
1888+
1889+func (m *ConcludeTransactionResponse) Reset() { *m = ConcludeTransactionResponse{} }
1890+func (m *ConcludeTransactionResponse) String() string { return "TODO" }
1891+
1892+
1893+
1894+// ReadTransactionRequest is the payload to ReadTransaction
1895+type ReadTransactionRequest struct {
1896+ EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
1897+ ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
1898+ Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
1899+ Dtid string `protobuf:"bytes,4,opt,name=dtid" json:"dtid,omitempty"`
1900+}
1901+
1902+func (m *ReadTransactionRequest) Reset() { *m = ReadTransactionRequest{} }
1903+func (m *ReadTransactionRequest) String() string { return "TODO" }
1904+
1905+
1906+
1907+func (m *ReadTransactionRequest) GetEffectiveCallerId() *vtrpc.CallerID {
1908+ if m != nil {
1909+ return m.EffectiveCallerId
1910+ }
1911+ return nil
1912+}
1913+
1914+func (m *ReadTransactionRequest) GetImmediateCallerId() *VTGateCallerID {
1915+ if m != nil {
1916+ return m.ImmediateCallerId
1917+ }
1918+ return nil
1919+}
1920+
1921+func (m *ReadTransactionRequest) GetTarget() *Target {
1922+ if m != nil {
1923+ return m.Target
1924+ }
1925+ return nil
1926+}
1927+
1928+func (m *ReadTransactionRequest) GetDtid() string {
1929+ if m != nil {
1930+ return m.Dtid
1931+ }
1932+ return ""
1933+}
1934+
1935+// ReadTransactionResponse is the returned value from ReadTransaction
1936+type ReadTransactionResponse struct {
1937+ Metadata *TransactionMetadata `protobuf:"bytes,1,opt,name=metadata" json:"metadata,omitempty"`
1938+}
1939+
1940+func (m *ReadTransactionResponse) Reset() { *m = ReadTransactionResponse{} }
1941+func (m *ReadTransactionResponse) String() string { return "TODO" }
1942+
1943+
1944+
1945+func (m *ReadTransactionResponse) GetMetadata() *TransactionMetadata {
1946+ if m != nil {
1947+ return m.Metadata
1948+ }
1949+ return nil
1950+}
1951+
1952+// BeginExecuteRequest is the payload to BeginExecute
1953+type BeginExecuteRequest struct {
1954+ EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
1955+ ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
1956+ Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
1957+ Query *BoundQuery `protobuf:"bytes,4,opt,name=query" json:"query,omitempty"`
1958+ Options *ExecuteOptions `protobuf:"bytes,5,opt,name=options" json:"options,omitempty"`
1959+}
1960+
1961+func (m *BeginExecuteRequest) Reset() { *m = BeginExecuteRequest{} }
1962+func (m *BeginExecuteRequest) String() string { return "TODO" }
1963+
1964+
1965+
1966+func (m *BeginExecuteRequest) GetEffectiveCallerId() *vtrpc.CallerID {
1967+ if m != nil {
1968+ return m.EffectiveCallerId
1969+ }
1970+ return nil
1971+}
1972+
1973+func (m *BeginExecuteRequest) GetImmediateCallerId() *VTGateCallerID {
1974+ if m != nil {
1975+ return m.ImmediateCallerId
1976+ }
1977+ return nil
1978+}
1979+
1980+func (m *BeginExecuteRequest) GetTarget() *Target {
1981+ if m != nil {
1982+ return m.Target
1983+ }
1984+ return nil
1985+}
1986+
1987+func (m *BeginExecuteRequest) GetQuery() *BoundQuery {
1988+ if m != nil {
1989+ return m.Query
1990+ }
1991+ return nil
1992+}
1993+
1994+func (m *BeginExecuteRequest) GetOptions() *ExecuteOptions {
1995+ if m != nil {
1996+ return m.Options
1997+ }
1998+ return nil
1999+}
2000+
2001+// BeginExecuteResponse is the returned value from BeginExecute
2002+type BeginExecuteResponse struct {
2003+ // error contains an application level error if necessary. Note the
2004+ // transaction_id may be set, even when an error is returned, if the begin
2005+ // worked but the execute failed.
2006+ Error *vtrpc.RPCError `protobuf:"bytes,1,opt,name=error" json:"error,omitempty"`
2007+ Result *QueryResult `protobuf:"bytes,2,opt,name=result" json:"result,omitempty"`
2008+ // transaction_id might be non-zero even if an error is present.
2009+ TransactionId int64 `protobuf:"varint,3,opt,name=transaction_id,json=transactionId" json:"transaction_id,omitempty"`
2010+}
2011+
2012+func (m *BeginExecuteResponse) Reset() { *m = BeginExecuteResponse{} }
2013+func (m *BeginExecuteResponse) String() string { return "TODO" }
2014+
2015+
2016+
2017+func (m *BeginExecuteResponse) GetError() *vtrpc.RPCError {
2018+ if m != nil {
2019+ return m.Error
2020+ }
2021+ return nil
2022+}
2023+
2024+func (m *BeginExecuteResponse) GetResult() *QueryResult {
2025+ if m != nil {
2026+ return m.Result
2027+ }
2028+ return nil
2029+}
2030+
2031+func (m *BeginExecuteResponse) GetTransactionId() int64 {
2032+ if m != nil {
2033+ return m.TransactionId
2034+ }
2035+ return 0
2036+}
2037+
2038+// BeginExecuteBatchRequest is the payload to BeginExecuteBatch
2039+type BeginExecuteBatchRequest struct {
2040+ EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
2041+ ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
2042+ Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
2043+ Queries []*BoundQuery `protobuf:"bytes,4,rep,name=queries" json:"queries,omitempty"`
2044+ AsTransaction bool `protobuf:"varint,5,opt,name=as_transaction,json=asTransaction" json:"as_transaction,omitempty"`
2045+ Options *ExecuteOptions `protobuf:"bytes,6,opt,name=options" json:"options,omitempty"`
2046+}
2047+
2048+func (m *BeginExecuteBatchRequest) Reset() { *m = BeginExecuteBatchRequest{} }
2049+func (m *BeginExecuteBatchRequest) String() string { return "TODO" }
2050+
2051+
2052+
2053+func (m *BeginExecuteBatchRequest) GetEffectiveCallerId() *vtrpc.CallerID {
2054+ if m != nil {
2055+ return m.EffectiveCallerId
2056+ }
2057+ return nil
2058+}
2059+
2060+func (m *BeginExecuteBatchRequest) GetImmediateCallerId() *VTGateCallerID {
2061+ if m != nil {
2062+ return m.ImmediateCallerId
2063+ }
2064+ return nil
2065+}
2066+
2067+func (m *BeginExecuteBatchRequest) GetTarget() *Target {
2068+ if m != nil {
2069+ return m.Target
2070+ }
2071+ return nil
2072+}
2073+
2074+func (m *BeginExecuteBatchRequest) GetQueries() []*BoundQuery {
2075+ if m != nil {
2076+ return m.Queries
2077+ }
2078+ return nil
2079+}
2080+
2081+func (m *BeginExecuteBatchRequest) GetAsTransaction() bool {
2082+ if m != nil {
2083+ return m.AsTransaction
2084+ }
2085+ return false
2086+}
2087+
2088+func (m *BeginExecuteBatchRequest) GetOptions() *ExecuteOptions {
2089+ if m != nil {
2090+ return m.Options
2091+ }
2092+ return nil
2093+}
2094+
2095+// BeginExecuteBatchResponse is the returned value from BeginExecuteBatch
2096+type BeginExecuteBatchResponse struct {
2097+ // error contains an application level error if necessary. Note the
2098+ // transaction_id may be set, even when an error is returned, if the begin
2099+ // worked but the execute failed.
2100+ Error *vtrpc.RPCError `protobuf:"bytes,1,opt,name=error" json:"error,omitempty"`
2101+ Results []*QueryResult `protobuf:"bytes,2,rep,name=results" json:"results,omitempty"`
2102+ // transaction_id might be non-zero even if an error is present.
2103+ TransactionId int64 `protobuf:"varint,3,opt,name=transaction_id,json=transactionId" json:"transaction_id,omitempty"`
2104+}
2105+
2106+func (m *BeginExecuteBatchResponse) Reset() { *m = BeginExecuteBatchResponse{} }
2107+func (m *BeginExecuteBatchResponse) String() string { return "TODO" }
2108+
2109+
2110+
2111+func (m *BeginExecuteBatchResponse) GetError() *vtrpc.RPCError {
2112+ if m != nil {
2113+ return m.Error
2114+ }
2115+ return nil
2116+}
2117+
2118+func (m *BeginExecuteBatchResponse) GetResults() []*QueryResult {
2119+ if m != nil {
2120+ return m.Results
2121+ }
2122+ return nil
2123+}
2124+
2125+func (m *BeginExecuteBatchResponse) GetTransactionId() int64 {
2126+ if m != nil {
2127+ return m.TransactionId
2128+ }
2129+ return 0
2130+}
2131+
2132+// MessageStreamRequest is the request payload for MessageStream.
2133+type MessageStreamRequest struct {
2134+ EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
2135+ ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
2136+ Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
2137+ // name is the message table name.
2138+ Name string `protobuf:"bytes,4,opt,name=name" json:"name,omitempty"`
2139+}
2140+
2141+func (m *MessageStreamRequest) Reset() { *m = MessageStreamRequest{} }
2142+func (m *MessageStreamRequest) String() string { return "TODO" }
2143+
2144+
2145+
2146+func (m *MessageStreamRequest) GetEffectiveCallerId() *vtrpc.CallerID {
2147+ if m != nil {
2148+ return m.EffectiveCallerId
2149+ }
2150+ return nil
2151+}
2152+
2153+func (m *MessageStreamRequest) GetImmediateCallerId() *VTGateCallerID {
2154+ if m != nil {
2155+ return m.ImmediateCallerId
2156+ }
2157+ return nil
2158+}
2159+
2160+func (m *MessageStreamRequest) GetTarget() *Target {
2161+ if m != nil {
2162+ return m.Target
2163+ }
2164+ return nil
2165+}
2166+
2167+func (m *MessageStreamRequest) GetName() string {
2168+ if m != nil {
2169+ return m.Name
2170+ }
2171+ return ""
2172+}
2173+
2174+// MessageStreamResponse is a response for MessageStream.
2175+type MessageStreamResponse struct {
2176+ Result *QueryResult `protobuf:"bytes,1,opt,name=result" json:"result,omitempty"`
2177+}
2178+
2179+func (m *MessageStreamResponse) Reset() { *m = MessageStreamResponse{} }
2180+func (m *MessageStreamResponse) String() string { return "TODO" }
2181+
2182+
2183+
2184+func (m *MessageStreamResponse) GetResult() *QueryResult {
2185+ if m != nil {
2186+ return m.Result
2187+ }
2188+ return nil
2189+}
2190+
2191+// MessageAckRequest is the request payload for MessageAck.
2192+type MessageAckRequest struct {
2193+ EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
2194+ ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
2195+ Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
2196+ // name is the message table name.
2197+ Name string `protobuf:"bytes,4,opt,name=name" json:"name,omitempty"`
2198+ Ids []*Value `protobuf:"bytes,5,rep,name=ids" json:"ids,omitempty"`
2199+}
2200+
2201+func (m *MessageAckRequest) Reset() { *m = MessageAckRequest{} }
2202+func (m *MessageAckRequest) String() string { return "TODO" }
2203+
2204+
2205+
2206+func (m *MessageAckRequest) GetEffectiveCallerId() *vtrpc.CallerID {
2207+ if m != nil {
2208+ return m.EffectiveCallerId
2209+ }
2210+ return nil
2211+}
2212+
2213+func (m *MessageAckRequest) GetImmediateCallerId() *VTGateCallerID {
2214+ if m != nil {
2215+ return m.ImmediateCallerId
2216+ }
2217+ return nil
2218+}
2219+
2220+func (m *MessageAckRequest) GetTarget() *Target {
2221+ if m != nil {
2222+ return m.Target
2223+ }
2224+ return nil
2225+}
2226+
2227+func (m *MessageAckRequest) GetName() string {
2228+ if m != nil {
2229+ return m.Name
2230+ }
2231+ return ""
2232+}
2233+
2234+func (m *MessageAckRequest) GetIds() []*Value {
2235+ if m != nil {
2236+ return m.Ids
2237+ }
2238+ return nil
2239+}
2240+
2241+// MessageAckResponse is the response for MessageAck.
2242+type MessageAckResponse struct {
2243+ // result contains the result of the ack operation.
2244+ // Since this acts like a DML, only
2245+ // RowsAffected is returned in the result.
2246+ Result *QueryResult `protobuf:"bytes,1,opt,name=result" json:"result,omitempty"`
2247+}
2248+
2249+func (m *MessageAckResponse) Reset() { *m = MessageAckResponse{} }
2250+func (m *MessageAckResponse) String() string { return "TODO" }
2251+
2252+
2253+
2254+func (m *MessageAckResponse) GetResult() *QueryResult {
2255+ if m != nil {
2256+ return m.Result
2257+ }
2258+ return nil
2259+}
2260+
2261+// SplitQueryRequest is the payload for SplitQuery sent by VTGate to a VTTablet.
2262+// See vtgate.SplitQueryRequest for more details.
2263+type SplitQueryRequest struct {
2264+ EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
2265+ ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
2266+ Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
2267+ Query *BoundQuery `protobuf:"bytes,4,opt,name=query" json:"query,omitempty"`
2268+ SplitColumn []string `protobuf:"bytes,5,rep,name=split_column,json=splitColumn" json:"split_column,omitempty"`
2269+ // Exactly one of the following must be nonzero.
2270+ SplitCount int64 `protobuf:"varint,6,opt,name=split_count,json=splitCount" json:"split_count,omitempty"`
2271+ NumRowsPerQueryPart int64 `protobuf:"varint,8,opt,name=num_rows_per_query_part,json=numRowsPerQueryPart" json:"num_rows_per_query_part,omitempty"`
2272+ Algorithm SplitQueryRequest_Algorithm `protobuf:"varint,9,opt,name=algorithm,enum=query.SplitQueryRequest_Algorithm" json:"algorithm,omitempty"`
2273+}
2274+
2275+func (m *SplitQueryRequest) Reset() { *m = SplitQueryRequest{} }
2276+func (m *SplitQueryRequest) String() string { return "TODO" }
2277+
2278+
2279+
2280+func (m *SplitQueryRequest) GetEffectiveCallerId() *vtrpc.CallerID {
2281+ if m != nil {
2282+ return m.EffectiveCallerId
2283+ }
2284+ return nil
2285+}
2286+
2287+func (m *SplitQueryRequest) GetImmediateCallerId() *VTGateCallerID {
2288+ if m != nil {
2289+ return m.ImmediateCallerId
2290+ }
2291+ return nil
2292+}
2293+
2294+func (m *SplitQueryRequest) GetTarget() *Target {
2295+ if m != nil {
2296+ return m.Target
2297+ }
2298+ return nil
2299+}
2300+
2301+func (m *SplitQueryRequest) GetQuery() *BoundQuery {
2302+ if m != nil {
2303+ return m.Query
2304+ }
2305+ return nil
2306+}
2307+
2308+func (m *SplitQueryRequest) GetSplitColumn() []string {
2309+ if m != nil {
2310+ return m.SplitColumn
2311+ }
2312+ return nil
2313+}
2314+
2315+func (m *SplitQueryRequest) GetSplitCount() int64 {
2316+ if m != nil {
2317+ return m.SplitCount
2318+ }
2319+ return 0
2320+}
2321+
2322+func (m *SplitQueryRequest) GetNumRowsPerQueryPart() int64 {
2323+ if m != nil {
2324+ return m.NumRowsPerQueryPart
2325+ }
2326+ return 0
2327+}
2328+
2329+func (m *SplitQueryRequest) GetAlgorithm() SplitQueryRequest_Algorithm {
2330+ if m != nil {
2331+ return m.Algorithm
2332+ }
2333+ return SplitQueryRequest_EQUAL_SPLITS
2334+}
2335+
2336+// QuerySplit represents one query to execute on the tablet
2337+type QuerySplit struct {
2338+ // query is the query to execute
2339+ Query *BoundQuery `protobuf:"bytes,1,opt,name=query" json:"query,omitempty"`
2340+ // row_count is the approximate row count the query will return
2341+ RowCount int64 `protobuf:"varint,2,opt,name=row_count,json=rowCount" json:"row_count,omitempty"`
2342+}
2343+
2344+func (m *QuerySplit) Reset() { *m = QuerySplit{} }
2345+func (m *QuerySplit) String() string { return "TODO" }
2346+
2347+
2348+
2349+func (m *QuerySplit) GetQuery() *BoundQuery {
2350+ if m != nil {
2351+ return m.Query
2352+ }
2353+ return nil
2354+}
2355+
2356+func (m *QuerySplit) GetRowCount() int64 {
2357+ if m != nil {
2358+ return m.RowCount
2359+ }
2360+ return 0
2361+}
2362+
2363+// SplitQueryResponse is returned by SplitQuery and represents all the queries
2364+// to execute in order to get the entire data set.
2365+type SplitQueryResponse struct {
2366+ Queries []*QuerySplit `protobuf:"bytes,1,rep,name=queries" json:"queries,omitempty"`
2367+}
2368+
2369+func (m *SplitQueryResponse) Reset() { *m = SplitQueryResponse{} }
2370+func (m *SplitQueryResponse) String() string { return "TODO" }
2371+
2372+
2373+
2374+func (m *SplitQueryResponse) GetQueries() []*QuerySplit {
2375+ if m != nil {
2376+ return m.Queries
2377+ }
2378+ return nil
2379+}
2380+
2381+// StreamHealthRequest is the payload for StreamHealth
2382+type StreamHealthRequest struct {
2383+}
2384+
2385+func (m *StreamHealthRequest) Reset() { *m = StreamHealthRequest{} }
2386+func (m *StreamHealthRequest) String() string { return "TODO" }
2387+
2388+
2389+
2390+// RealtimeStats contains information about the tablet status
2391+type RealtimeStats struct {
2392+ // health_error is the last error we got from health check,
2393+ // or empty is the server is healthy. This is used for subset selection,
2394+ // we do not send queries to servers that are not healthy.
2395+ HealthError string `protobuf:"bytes,1,opt,name=health_error,json=healthError" json:"health_error,omitempty"`
2396+ // seconds_behind_master is populated for slaves only. It indicates
2397+ // how far behind on (MySQL) replication a slave currently is. It is used
2398+ // by clients for subset selection (so we don't try to send traffic
2399+ // to tablets that are too far behind).
2400+ // NOTE: This field must not be evaluated if "health_error" is not empty.
2401+ // TODO(mberlin): Let's switch it to int64 instead?
2402+ SecondsBehindMaster uint32 `protobuf:"varint,2,opt,name=seconds_behind_master,json=secondsBehindMaster" json:"seconds_behind_master,omitempty"`
2403+ // bin_log_players_count is the number of currently running binlog players.
2404+ // if the value is 0, it means that filtered replication is currently not
2405+ // running on the tablet. If >0, filtered replication is running.
2406+ // NOTE: This field must not be evaluated if "health_error" is not empty.
2407+ BinlogPlayersCount int32 `protobuf:"varint,3,opt,name=binlog_players_count,json=binlogPlayersCount" json:"binlog_players_count,omitempty"`
2408+ // seconds_behind_master_filtered_replication is populated for the receiving
2409+ // master of an ongoing filtered replication only.
2410+ // It specifies how far the receiving master lags behind the sending master.
2411+ // NOTE: This field must not be evaluated if "health_error" is not empty.
2412+ // NOTE: This field must not be evaluated if "bin_log_players_count" is 0.
2413+ SecondsBehindMasterFilteredReplication int64 `protobuf:"varint,4,opt,name=seconds_behind_master_filtered_replication,json=secondsBehindMasterFilteredReplication" json:"seconds_behind_master_filtered_replication,omitempty"`
2414+ // cpu_usage is used for load-based balancing
2415+ CpuUsage float64 `protobuf:"fixed64,5,opt,name=cpu_usage,json=cpuUsage" json:"cpu_usage,omitempty"`
2416+ // qps is the average QPS (queries per second) rate in the last XX seconds
2417+ // where XX is usually 60 (See query_service_stats.go).
2418+ Qps float64 `protobuf:"fixed64,6,opt,name=qps" json:"qps,omitempty"`
2419+}
2420+
2421+func (m *RealtimeStats) Reset() { *m = RealtimeStats{} }
2422+func (m *RealtimeStats) String() string { return "TODO" }
2423+
2424+
2425+
2426+func (m *RealtimeStats) GetHealthError() string {
2427+ if m != nil {
2428+ return m.HealthError
2429+ }
2430+ return ""
2431+}
2432+
2433+func (m *RealtimeStats) GetSecondsBehindMaster() uint32 {
2434+ if m != nil {
2435+ return m.SecondsBehindMaster
2436+ }
2437+ return 0
2438+}
2439+
2440+func (m *RealtimeStats) GetBinlogPlayersCount() int32 {
2441+ if m != nil {
2442+ return m.BinlogPlayersCount
2443+ }
2444+ return 0
2445+}
2446+
2447+func (m *RealtimeStats) GetSecondsBehindMasterFilteredReplication() int64 {
2448+ if m != nil {
2449+ return m.SecondsBehindMasterFilteredReplication
2450+ }
2451+ return 0
2452+}
2453+
2454+func (m *RealtimeStats) GetCpuUsage() float64 {
2455+ if m != nil {
2456+ return m.CpuUsage
2457+ }
2458+ return 0
2459+}
2460+
2461+func (m *RealtimeStats) GetQps() float64 {
2462+ if m != nil {
2463+ return m.Qps
2464+ }
2465+ return 0
2466+}
2467+
2468+// StreamHealthResponse is streamed by StreamHealth on a regular basis
2469+type StreamHealthResponse struct {
2470+ // target is the current server type. Only queries with that exact Target
2471+ // record will be accepted.
2472+ Target *Target `protobuf:"bytes,1,opt,name=target" json:"target,omitempty"`
2473+ // serving is true iff the tablet is serving. A tablet may not be serving
2474+ // if filtered replication is enabled on a master for instance,
2475+ // or if a replica should not be used because the keyspace is being resharded.
2476+ Serving bool `protobuf:"varint,2,opt,name=serving" json:"serving,omitempty"`
2477+ // tablet_externally_reparented_timestamp can be interpreted as the last time
2478+ // we knew that this tablet was the MASTER of this shard.
2479+ //
2480+ // It is used by vtgate when determining the current MASTER of a shard.
2481+ // If vtgate sees more than one MASTER tablet, this timestamp is used
2482+ // as tiebreaker where the MASTER with the highest timestamp wins.
2483+ // Another usage of this timestamp is in go/vt/vtgate/buffer to detect the end
2484+ // of a reparent (failover) and stop buffering.
2485+ //
2486+ // In practice, this field is set to:
2487+ // a) the last time the RPC tabletmanager.TabletExternallyReparented was
2488+ // called on this tablet (usually done by an external failover tool e.g.
2489+ // Orchestrator). The failover tool can call this as long as we are the
2490+ // master i.e. even ages after the last reparent occurred.
2491+ // OR
2492+ // b) the last time an active reparent was executed through a vtctl command
2493+ // (InitShardMaster, PlannedReparentShard, EmergencyReparentShard)
2494+ // OR
2495+ // c) the last time vttablet was started and it initialized its tablet type
2496+ // as MASTER because it was recorded as the shard's current master in the
2497+ // topology (see go/vt/vttablet/tabletmanager/init_tablet.go)
2498+ // OR
2499+ // d) 0 if the vttablet was never a MASTER.
2500+ TabletExternallyReparentedTimestamp int64 `protobuf:"varint,3,opt,name=tablet_externally_reparented_timestamp,json=tabletExternallyReparentedTimestamp" json:"tablet_externally_reparented_timestamp,omitempty"`
2501+ // realtime_stats contains information about the tablet status
2502+ RealtimeStats *RealtimeStats `protobuf:"bytes,4,opt,name=realtime_stats,json=realtimeStats" json:"realtime_stats,omitempty"`
2503+ // tablet_alias is the alias of the sending tablet. The discovery/healthcheck.go
2504+ // code uses it to verify that it's talking to the correct tablet and that it
2505+ // hasn't changed in the meantime e.g. due to tablet restarts where ports or
2506+ // ips have been reused but assigned differently.
2507+ TabletAlias *topodata.TabletAlias `protobuf:"bytes,5,opt,name=tablet_alias,json=tabletAlias" json:"tablet_alias,omitempty"`
2508+}
2509+
2510+func (m *StreamHealthResponse) Reset() { *m = StreamHealthResponse{} }
2511+func (m *StreamHealthResponse) String() string { return "TODO" }
2512+
2513+
2514+
2515+func (m *StreamHealthResponse) GetTarget() *Target {
2516+ if m != nil {
2517+ return m.Target
2518+ }
2519+ return nil
2520+}
2521+
2522+func (m *StreamHealthResponse) GetServing() bool {
2523+ if m != nil {
2524+ return m.Serving
2525+ }
2526+ return false
2527+}
2528+
2529+func (m *StreamHealthResponse) GetTabletExternallyReparentedTimestamp() int64 {
2530+ if m != nil {
2531+ return m.TabletExternallyReparentedTimestamp
2532+ }
2533+ return 0
2534+}
2535+
2536+func (m *StreamHealthResponse) GetRealtimeStats() *RealtimeStats {
2537+ if m != nil {
2538+ return m.RealtimeStats
2539+ }
2540+ return nil
2541+}
2542+
2543+func (m *StreamHealthResponse) GetTabletAlias() *topodata.TabletAlias {
2544+ if m != nil {
2545+ return m.TabletAlias
2546+ }
2547+ return nil
2548+}
2549+
2550+// UpdateStreamRequest is the payload for UpdateStream. At most one of
2551+// position and timestamp can be set. If neither is set, we will start
2552+// streaming from the current binlog position.
2553+type UpdateStreamRequest struct {
2554+ EffectiveCallerId *vtrpc.CallerID `protobuf:"bytes,1,opt,name=effective_caller_id,json=effectiveCallerId" json:"effective_caller_id,omitempty"`
2555+ ImmediateCallerId *VTGateCallerID `protobuf:"bytes,2,opt,name=immediate_caller_id,json=immediateCallerId" json:"immediate_caller_id,omitempty"`
2556+ Target *Target `protobuf:"bytes,3,opt,name=target" json:"target,omitempty"`
2557+ // If position is set, we will start the streaming from that replication
2558+ // position. Incompatible with timestamp.
2559+ Position string `protobuf:"bytes,4,opt,name=position" json:"position,omitempty"`
2560+ // If timestamp is set, we will start the streaming from the first
2561+ // event in the binlogs that have that timestamp. Incompatible with position.
2562+ Timestamp int64 `protobuf:"varint,5,opt,name=timestamp" json:"timestamp,omitempty"`
2563+}
2564+
2565+func (m *UpdateStreamRequest) Reset() { *m = UpdateStreamRequest{} }
2566+func (m *UpdateStreamRequest) String() string { return "TODO" }
2567+
2568+
2569+
2570+func (m *UpdateStreamRequest) GetEffectiveCallerId() *vtrpc.CallerID {
2571+ if m != nil {
2572+ return m.EffectiveCallerId
2573+ }
2574+ return nil
2575+}
2576+
2577+func (m *UpdateStreamRequest) GetImmediateCallerId() *VTGateCallerID {
2578+ if m != nil {
2579+ return m.ImmediateCallerId
2580+ }
2581+ return nil
2582+}
2583+
2584+func (m *UpdateStreamRequest) GetTarget() *Target {
2585+ if m != nil {
2586+ return m.Target
2587+ }
2588+ return nil
2589+}
2590+
2591+func (m *UpdateStreamRequest) GetPosition() string {
2592+ if m != nil {
2593+ return m.Position
2594+ }
2595+ return ""
2596+}
2597+
2598+func (m *UpdateStreamRequest) GetTimestamp() int64 {
2599+ if m != nil {
2600+ return m.Timestamp
2601+ }
2602+ return 0
2603+}
2604+
2605+// UpdateStreamResponse is returned by UpdateStream
2606+type UpdateStreamResponse struct {
2607+ Event *StreamEvent `protobuf:"bytes,1,opt,name=event" json:"event,omitempty"`
2608+}
2609+
2610+func (m *UpdateStreamResponse) Reset() { *m = UpdateStreamResponse{} }
2611+func (m *UpdateStreamResponse) String() string { return "TODO" }
2612+
2613+
2614+
2615+func (m *UpdateStreamResponse) GetEvent() *StreamEvent {
2616+ if m != nil {
2617+ return m.Event
2618+ }
2619+ return nil
2620+}
2621+
2622+// TransactionMetadata contains the metadata for a distributed transaction.
2623+type TransactionMetadata struct {
2624+ Dtid string `protobuf:"bytes,1,opt,name=dtid" json:"dtid,omitempty"`
2625+ State TransactionState `protobuf:"varint,2,opt,name=state,enum=query.TransactionState" json:"state,omitempty"`
2626+ TimeCreated int64 `protobuf:"varint,3,opt,name=time_created,json=timeCreated" json:"time_created,omitempty"`
2627+ Participants []*Target `protobuf:"bytes,4,rep,name=participants" json:"participants,omitempty"`
2628+}
2629+
2630+func (m *TransactionMetadata) Reset() { *m = TransactionMetadata{} }
2631+func (m *TransactionMetadata) String() string { return "TODO" }
2632+
2633+
2634+
2635+func (m *TransactionMetadata) GetDtid() string {
2636+ if m != nil {
2637+ return m.Dtid
2638+ }
2639+ return ""
2640+}
2641+
2642+func (m *TransactionMetadata) GetState() TransactionState {
2643+ if m != nil {
2644+ return m.State
2645+ }
2646+ return TransactionState_UNKNOWN
2647+}
2648+
2649+func (m *TransactionMetadata) GetTimeCreated() int64 {
2650+ if m != nil {
2651+ return m.TimeCreated
2652+ }
2653+ return 0
2654+}
2655+
2656+func (m *TransactionMetadata) GetParticipants() []*Target {
2657+ if m != nil {
2658+ return m.Participants
2659+ }
2660+ return nil
2661+}
2662+
2663+func init() {
2664+ proto.RegisterType((*Target)(nil), "query.Target")
2665+ proto.RegisterType((*VTGateCallerID)(nil), "query.VTGateCallerID")
2666+ proto.RegisterType((*EventToken)(nil), "query.EventToken")
2667+ proto.RegisterType((*Value)(nil), "query.Value")
2668+ proto.RegisterType((*BindVariable)(nil), "query.BindVariable")
2669+ proto.RegisterType((*BoundQuery)(nil), "query.BoundQuery")
2670+ proto.RegisterType((*ExecuteOptions)(nil), "query.ExecuteOptions")
2671+ proto.RegisterType((*Field)(nil), "query.Field")
2672+ proto.RegisterType((*Row)(nil), "query.Row")
2673+ proto.RegisterType((*ResultExtras)(nil), "query.ResultExtras")
2674+ proto.RegisterType((*QueryResult)(nil), "query.QueryResult")
2675+ proto.RegisterType((*StreamEvent)(nil), "query.StreamEvent")
2676+ proto.RegisterType((*StreamEvent_Statement)(nil), "query.StreamEvent.Statement")
2677+ proto.RegisterType((*ExecuteRequest)(nil), "query.ExecuteRequest")
2678+ proto.RegisterType((*ExecuteResponse)(nil), "query.ExecuteResponse")
2679+ proto.RegisterType((*ResultWithError)(nil), "query.ResultWithError")
2680+ proto.RegisterType((*ExecuteBatchRequest)(nil), "query.ExecuteBatchRequest")
2681+ proto.RegisterType((*ExecuteBatchResponse)(nil), "query.ExecuteBatchResponse")
2682+ proto.RegisterType((*StreamExecuteRequest)(nil), "query.StreamExecuteRequest")
2683+ proto.RegisterType((*StreamExecuteResponse)(nil), "query.StreamExecuteResponse")
2684+ proto.RegisterType((*BeginRequest)(nil), "query.BeginRequest")
2685+ proto.RegisterType((*BeginResponse)(nil), "query.BeginResponse")
2686+ proto.RegisterType((*CommitRequest)(nil), "query.CommitRequest")
2687+ proto.RegisterType((*CommitResponse)(nil), "query.CommitResponse")
2688+ proto.RegisterType((*RollbackRequest)(nil), "query.RollbackRequest")
2689+ proto.RegisterType((*RollbackResponse)(nil), "query.RollbackResponse")
2690+ proto.RegisterType((*PrepareRequest)(nil), "query.PrepareRequest")
2691+ proto.RegisterType((*PrepareResponse)(nil), "query.PrepareResponse")
2692+ proto.RegisterType((*CommitPreparedRequest)(nil), "query.CommitPreparedRequest")
2693+ proto.RegisterType((*CommitPreparedResponse)(nil), "query.CommitPreparedResponse")
2694+ proto.RegisterType((*RollbackPreparedRequest)(nil), "query.RollbackPreparedRequest")
2695+ proto.RegisterType((*RollbackPreparedResponse)(nil), "query.RollbackPreparedResponse")
2696+ proto.RegisterType((*CreateTransactionRequest)(nil), "query.CreateTransactionRequest")
2697+ proto.RegisterType((*CreateTransactionResponse)(nil), "query.CreateTransactionResponse")
2698+ proto.RegisterType((*StartCommitRequest)(nil), "query.StartCommitRequest")
2699+ proto.RegisterType((*StartCommitResponse)(nil), "query.StartCommitResponse")
2700+ proto.RegisterType((*SetRollbackRequest)(nil), "query.SetRollbackRequest")
2701+ proto.RegisterType((*SetRollbackResponse)(nil), "query.SetRollbackResponse")
2702+ proto.RegisterType((*ConcludeTransactionRequest)(nil), "query.ConcludeTransactionRequest")
2703+ proto.RegisterType((*ConcludeTransactionResponse)(nil), "query.ConcludeTransactionResponse")
2704+ proto.RegisterType((*ReadTransactionRequest)(nil), "query.ReadTransactionRequest")
2705+ proto.RegisterType((*ReadTransactionResponse)(nil), "query.ReadTransactionResponse")
2706+ proto.RegisterType((*BeginExecuteRequest)(nil), "query.BeginExecuteRequest")
2707+ proto.RegisterType((*BeginExecuteResponse)(nil), "query.BeginExecuteResponse")
2708+ proto.RegisterType((*BeginExecuteBatchRequest)(nil), "query.BeginExecuteBatchRequest")
2709+ proto.RegisterType((*BeginExecuteBatchResponse)(nil), "query.BeginExecuteBatchResponse")
2710+ proto.RegisterType((*MessageStreamRequest)(nil), "query.MessageStreamRequest")
2711+ proto.RegisterType((*MessageStreamResponse)(nil), "query.MessageStreamResponse")
2712+ proto.RegisterType((*MessageAckRequest)(nil), "query.MessageAckRequest")
2713+ proto.RegisterType((*MessageAckResponse)(nil), "query.MessageAckResponse")
2714+ proto.RegisterType((*SplitQueryRequest)(nil), "query.SplitQueryRequest")
2715+ proto.RegisterType((*QuerySplit)(nil), "query.QuerySplit")
2716+ proto.RegisterType((*SplitQueryResponse)(nil), "query.SplitQueryResponse")
2717+ proto.RegisterType((*StreamHealthRequest)(nil), "query.StreamHealthRequest")
2718+ proto.RegisterType((*RealtimeStats)(nil), "query.RealtimeStats")
2719+ proto.RegisterType((*StreamHealthResponse)(nil), "query.StreamHealthResponse")
2720+ proto.RegisterType((*UpdateStreamRequest)(nil), "query.UpdateStreamRequest")
2721+ proto.RegisterType((*UpdateStreamResponse)(nil), "query.UpdateStreamResponse")
2722+ proto.RegisterType((*TransactionMetadata)(nil), "query.TransactionMetadata")
2723+ proto.RegisterEnum("query.MySqlFlag", MySqlFlag_name, MySqlFlag_value)
2724+ proto.RegisterEnum("query.Flag", Flag_name, Flag_value)
2725+ proto.RegisterEnum("query.Type", Type_name, Type_value)
2726+ proto.RegisterEnum("query.TransactionState", TransactionState_name, TransactionState_value)
2727+ proto.RegisterEnum("query.ExecuteOptions_IncludedFields", ExecuteOptions_IncludedFields_name, ExecuteOptions_IncludedFields_value)
2728+ proto.RegisterEnum("query.ExecuteOptions_Workload", ExecuteOptions_Workload_name, ExecuteOptions_Workload_value)
2729+ proto.RegisterEnum("query.ExecuteOptions_TransactionIsolation", ExecuteOptions_TransactionIsolation_name, ExecuteOptions_TransactionIsolation_value)
2730+ proto.RegisterEnum("query.StreamEvent_Statement_Category", StreamEvent_Statement_Category_name, StreamEvent_Statement_Category_value)
2731+ proto.RegisterEnum("query.SplitQueryRequest_Algorithm", SplitQueryRequest_Algorithm_name, SplitQueryRequest_Algorithm_value)
2732+}
2733+
2734+*/
vendor/github.com/xwb1989/sqlparser/dependency/sqltypes/bind_variables.go+266−0
@@ -0,0 +1,266 @@
1+/*
2+Copyright 2017 Google Inc.
3+
4+Licensed under the Apache License, Version 2.0 (the "License");
5+you may not use this file except in compliance with the License.
6+You may obtain a copy of the License at
7+
8+ http://www.apache.org/licenses/LICENSE-2.0
9+
10+Unless required by applicable law or agreed to in writing, software
11+distributed under the License is distributed on an "AS IS" BASIS,
12+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+See the License for the specific language governing permissions and
14+limitations under the License.
15+*/
16+
17+package sqltypes
18+
19+import (
20+ "errors"
21+ "fmt"
22+ "reflect"
23+ "strconv"
24+
25+ "github.com/xwb1989/sqlparser/dependency/querypb"
26+)
27+
28+// NullBindVariable is a bindvar with NULL value.
29+var NullBindVariable = &querypb.BindVariable{Type: querypb.Type_NULL_TYPE}
30+
31+// ValueToProto converts Value to a *querypb.Value.
32+func ValueToProto(v Value) *querypb.Value {
33+ return &querypb.Value{Type: v.typ, Value: v.val}
34+}
35+
36+// ProtoToValue converts a *querypb.Value to a Value.
37+func ProtoToValue(v *querypb.Value) Value {
38+ return MakeTrusted(v.Type, v.Value)
39+}
40+
41+// BuildBindVariables builds a map[string]*querypb.BindVariable from a map[string]interface{}.
42+func BuildBindVariables(in map[string]interface{}) (map[string]*querypb.BindVariable, error) {
43+ if len(in) == 0 {
44+ return nil, nil
45+ }
46+
47+ out := make(map[string]*querypb.BindVariable, len(in))
48+ for k, v := range in {
49+ bv, err := BuildBindVariable(v)
50+ if err != nil {
51+ return nil, fmt.Errorf("%s: %v", k, err)
52+ }
53+ out[k] = bv
54+ }
55+ return out, nil
56+}
57+
58+// Int32BindVariable converts an int32 to a bind var.
59+func Int32BindVariable(v int32) *querypb.BindVariable {
60+ return ValueBindVariable(NewInt32(v))
61+}
62+
63+// Int64BindVariable converts an int64 to a bind var.
64+func Int64BindVariable(v int64) *querypb.BindVariable {
65+ return ValueBindVariable(NewInt64(v))
66+}
67+
68+// Uint64BindVariable converts a uint64 to a bind var.
69+func Uint64BindVariable(v uint64) *querypb.BindVariable {
70+ return ValueBindVariable(NewUint64(v))
71+}
72+
73+// Float64BindVariable converts a float64 to a bind var.
74+func Float64BindVariable(v float64) *querypb.BindVariable {
75+ return ValueBindVariable(NewFloat64(v))
76+}
77+
78+// StringBindVariable converts a string to a bind var.
79+func StringBindVariable(v string) *querypb.BindVariable {
80+ return ValueBindVariable(NewVarChar(v))
81+}
82+
83+// BytesBindVariable converts a []byte to a bind var.
84+func BytesBindVariable(v []byte) *querypb.BindVariable {
85+ return &querypb.BindVariable{Type: VarBinary, Value: v}
86+}
87+
88+// ValueBindVariable converts a Value to a bind var.
89+func ValueBindVariable(v Value) *querypb.BindVariable {
90+ return &querypb.BindVariable{Type: v.typ, Value: v.val}
91+}
92+
93+// BuildBindVariable builds a *querypb.BindVariable from a valid input type.
94+func BuildBindVariable(v interface{}) (*querypb.BindVariable, error) {
95+ switch v := v.(type) {
96+ case string:
97+ return StringBindVariable(v), nil
98+ case []byte:
99+ return BytesBindVariable(v), nil
100+ case int:
101+ return &querypb.BindVariable{
102+ Type: querypb.Type_INT64,
103+ Value: strconv.AppendInt(nil, int64(v), 10),
104+ }, nil
105+ case int64:
106+ return Int64BindVariable(v), nil
107+ case uint64:
108+ return Uint64BindVariable(v), nil
109+ case float64:
110+ return Float64BindVariable(v), nil
111+ case nil:
112+ return NullBindVariable, nil
113+ case Value:
114+ return ValueBindVariable(v), nil
115+ case *querypb.BindVariable:
116+ return v, nil
117+ case []interface{}:
118+ bv := &querypb.BindVariable{
119+ Type: querypb.Type_TUPLE,
120+ Values: make([]*querypb.Value, len(v)),
121+ }
122+ values := make([]querypb.Value, len(v))
123+ for i, lv := range v {
124+ lbv, err := BuildBindVariable(lv)
125+ if err != nil {
126+ return nil, err
127+ }
128+ values[i].Type = lbv.Type
129+ values[i].Value = lbv.Value
130+ bv.Values[i] = &values[i]
131+ }
132+ return bv, nil
133+ case []string:
134+ bv := &querypb.BindVariable{
135+ Type: querypb.Type_TUPLE,
136+ Values: make([]*querypb.Value, len(v)),
137+ }
138+ values := make([]querypb.Value, len(v))
139+ for i, lv := range v {
140+ values[i].Type = querypb.Type_VARCHAR
141+ values[i].Value = []byte(lv)
142+ bv.Values[i] = &values[i]
143+ }
144+ return bv, nil
145+ case [][]byte:
146+ bv := &querypb.BindVariable{
147+ Type: querypb.Type_TUPLE,
148+ Values: make([]*querypb.Value, len(v)),
149+ }
150+ values := make([]querypb.Value, len(v))
151+ for i, lv := range v {
152+ values[i].Type = querypb.Type_VARBINARY
153+ values[i].Value = lv
154+ bv.Values[i] = &values[i]
155+ }
156+ return bv, nil
157+ case []int:
158+ bv := &querypb.BindVariable{
159+ Type: querypb.Type_TUPLE,
160+ Values: make([]*querypb.Value, len(v)),
161+ }
162+ values := make([]querypb.Value, len(v))
163+ for i, lv := range v {
164+ values[i].Type = querypb.Type_INT64
165+ values[i].Value = strconv.AppendInt(nil, int64(lv), 10)
166+ bv.Values[i] = &values[i]
167+ }
168+ return bv, nil
169+ case []int64:
170+ bv := &querypb.BindVariable{
171+ Type: querypb.Type_TUPLE,
172+ Values: make([]*querypb.Value, len(v)),
173+ }
174+ values := make([]querypb.Value, len(v))
175+ for i, lv := range v {
176+ values[i].Type = querypb.Type_INT64
177+ values[i].Value = strconv.AppendInt(nil, lv, 10)
178+ bv.Values[i] = &values[i]
179+ }
180+ return bv, nil
181+ case []uint64:
182+ bv := &querypb.BindVariable{
183+ Type: querypb.Type_TUPLE,
184+ Values: make([]*querypb.Value, len(v)),
185+ }
186+ values := make([]querypb.Value, len(v))
187+ for i, lv := range v {
188+ values[i].Type = querypb.Type_UINT64
189+ values[i].Value = strconv.AppendUint(nil, lv, 10)
190+ bv.Values[i] = &values[i]
191+ }
192+ return bv, nil
193+ case []float64:
194+ bv := &querypb.BindVariable{
195+ Type: querypb.Type_TUPLE,
196+ Values: make([]*querypb.Value, len(v)),
197+ }
198+ values := make([]querypb.Value, len(v))
199+ for i, lv := range v {
200+ values[i].Type = querypb.Type_FLOAT64
201+ values[i].Value = strconv.AppendFloat(nil, lv, 'g', -1, 64)
202+ bv.Values[i] = &values[i]
203+ }
204+ return bv, nil
205+ }
206+ return nil, fmt.Errorf("type %T not supported as bind var: %v", v, v)
207+}
208+
209+// ValidateBindVariables validates a map[string]*querypb.BindVariable.
210+func ValidateBindVariables(bv map[string]*querypb.BindVariable) error {
211+ for k, v := range bv {
212+ if err := ValidateBindVariable(v); err != nil {
213+ return fmt.Errorf("%s: %v", k, err)
214+ }
215+ }
216+ return nil
217+}
218+
219+// ValidateBindVariable returns an error if the bind variable has inconsistent
220+// fields.
221+func ValidateBindVariable(bv *querypb.BindVariable) error {
222+ if bv == nil {
223+ return errors.New("bind variable is nil")
224+ }
225+
226+ if bv.Type == querypb.Type_TUPLE {
227+ if len(bv.Values) == 0 {
228+ return errors.New("empty tuple is not allowed")
229+ }
230+ for _, val := range bv.Values {
231+ if val.Type == querypb.Type_TUPLE {
232+ return errors.New("tuple not allowed inside another tuple")
233+ }
234+ if err := ValidateBindVariable(&querypb.BindVariable{Type: val.Type, Value: val.Value}); err != nil {
235+ return err
236+ }
237+ }
238+ return nil
239+ }
240+
241+ // If NewValue succeeds, the value is valid.
242+ _, err := NewValue(bv.Type, bv.Value)
243+ return err
244+}
245+
246+// BindVariableToValue converts a bind var into a Value.
247+func BindVariableToValue(bv *querypb.BindVariable) (Value, error) {
248+ if bv.Type == querypb.Type_TUPLE {
249+ return NULL, errors.New("cannot convert a TUPLE bind var into a value")
250+ }
251+ return MakeTrusted(bv.Type, bv.Value), nil
252+}
253+
254+// BindVariablesEqual compares two maps of bind variables.
255+func BindVariablesEqual(x, y map[string]*querypb.BindVariable) bool {
256+ return reflect.DeepEqual(&querypb.BoundQuery{BindVariables: x}, &querypb.BoundQuery{BindVariables: y})
257+}
258+
259+// CopyBindVariables returns a shallow-copy of the given bindVariables map.
260+func CopyBindVariables(bindVariables map[string]*querypb.BindVariable) map[string]*querypb.BindVariable {
261+ result := make(map[string]*querypb.BindVariable, len(bindVariables))
262+ for key, value := range bindVariables {
263+ result[key] = value
264+ }
265+ return result
266+}
vendor/github.com/xwb1989/sqlparser/dependency/sqltypes/plan_value.go+259−0
@@ -0,0 +1,259 @@
1+/*
2+Copyright 2017 Google Inc.
3+
4+Licensed under the Apache License, Version 2.0 (the "License");
5+you may not use this file except in compliance with the License.
6+You may obtain a copy of the License at
7+
8+ http://www.apache.org/licenses/LICENSE-2.0
9+
10+Unless required by applicable law or agreed to in writing, software
11+distributed under the License is distributed on an "AS IS" BASIS,
12+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+See the License for the specific language governing permissions and
14+limitations under the License.
15+*/
16+
17+package sqltypes
18+
19+import (
20+ "encoding/json"
21+ "errors"
22+ "fmt"
23+
24+ "github.com/xwb1989/sqlparser/dependency/querypb"
25+)
26+
27+// PlanValue represents a value or a list of values for
28+// a column that will later be resolved using bind vars and used
29+// to perform plan actions like generating the final query or
30+// deciding on a route.
31+//
32+// Plan values are typically used as a slice ([]planValue)
33+// where each entry is for one column. For situations where
34+// the required output is a list of rows (like in the case
35+// of multi-value inserts), the representation is pivoted.
36+// For example, a statement like this:
37+// INSERT INTO t VALUES (1, 2), (3, 4)
38+// will be represented as follows:
39+// []PlanValue{
40+// Values: {1, 3},
41+// Values: {2, 4},
42+// }
43+//
44+// For WHERE clause items that contain a combination of
45+// equality expressions and IN clauses like this:
46+// WHERE pk1 = 1 AND pk2 IN (2, 3, 4)
47+// The plan values will be represented as follows:
48+// []PlanValue{
49+// Value: 1,
50+// Values: {2, 3, 4},
51+// }
52+// When converted into rows, columns with single values
53+// are replicated as the same for all rows:
54+// [][]Value{
55+// {1, 2},
56+// {1, 3},
57+// {1, 4},
58+// }
59+type PlanValue struct {
60+ Key string
61+ Value Value
62+ ListKey string
63+ Values []PlanValue
64+}
65+
66+// IsNull returns true if the PlanValue is NULL.
67+func (pv PlanValue) IsNull() bool {
68+ return pv.Key == "" && pv.Value.IsNull() && pv.ListKey == "" && pv.Values == nil
69+}
70+
71+// IsList returns true if the PlanValue is a list.
72+func (pv PlanValue) IsList() bool {
73+ return pv.ListKey != "" || pv.Values != nil
74+}
75+
76+// ResolveValue resolves a PlanValue as a single value based on the supplied bindvars.
77+func (pv PlanValue) ResolveValue(bindVars map[string]*querypb.BindVariable) (Value, error) {
78+ switch {
79+ case pv.Key != "":
80+ bv, err := pv.lookupValue(bindVars)
81+ if err != nil {
82+ return NULL, err
83+ }
84+ return MakeTrusted(bv.Type, bv.Value), nil
85+ case !pv.Value.IsNull():
86+ return pv.Value, nil
87+ case pv.ListKey != "" || pv.Values != nil:
88+ // This code is unreachable because the parser does not allow
89+ // multi-value constructs where a single value is expected.
90+ return NULL, errors.New("a list was supplied where a single value was expected")
91+ }
92+ return NULL, nil
93+}
94+
95+func (pv PlanValue) lookupValue(bindVars map[string]*querypb.BindVariable) (*querypb.BindVariable, error) {
96+ bv, ok := bindVars[pv.Key]
97+ if !ok {
98+ return nil, fmt.Errorf("missing bind var %s", pv.Key)
99+ }
100+ if bv.Type == querypb.Type_TUPLE {
101+ return nil, fmt.Errorf("TUPLE was supplied for single value bind var %s", pv.ListKey)
102+ }
103+ return bv, nil
104+}
105+
106+// ResolveList resolves a PlanValue as a list of values based on the supplied bindvars.
107+func (pv PlanValue) ResolveList(bindVars map[string]*querypb.BindVariable) ([]Value, error) {
108+ switch {
109+ case pv.ListKey != "":
110+ bv, err := pv.lookupList(bindVars)
111+ if err != nil {
112+ return nil, err
113+ }
114+ values := make([]Value, 0, len(bv.Values))
115+ for _, val := range bv.Values {
116+ values = append(values, MakeTrusted(val.Type, val.Value))
117+ }
118+ return values, nil
119+ case pv.Values != nil:
120+ values := make([]Value, 0, len(pv.Values))
121+ for _, val := range pv.Values {
122+ v, err := val.ResolveValue(bindVars)
123+ if err != nil {
124+ return nil, err
125+ }
126+ values = append(values, v)
127+ }
128+ return values, nil
129+ }
130+ // This code is unreachable because the parser does not allow
131+ // single value constructs where multiple values are expected.
132+ return nil, errors.New("a single value was supplied where a list was expected")
133+}
134+
135+func (pv PlanValue) lookupList(bindVars map[string]*querypb.BindVariable) (*querypb.BindVariable, error) {
136+ bv, ok := bindVars[pv.ListKey]
137+ if !ok {
138+ return nil, fmt.Errorf("missing bind var %s", pv.ListKey)
139+ }
140+ if bv.Type != querypb.Type_TUPLE {
141+ return nil, fmt.Errorf("single value was supplied for TUPLE bind var %s", pv.ListKey)
142+ }
143+ return bv, nil
144+}
145+
146+// MarshalJSON should be used only for testing.
147+func (pv PlanValue) MarshalJSON() ([]byte, error) {
148+ switch {
149+ case pv.Key != "":
150+ return json.Marshal(":" + pv.Key)
151+ case !pv.Value.IsNull():
152+ if pv.Value.IsIntegral() {
153+ return pv.Value.ToBytes(), nil
154+ }
155+ return json.Marshal(pv.Value.ToString())
156+ case pv.ListKey != "":
157+ return json.Marshal("::" + pv.ListKey)
158+ case pv.Values != nil:
159+ return json.Marshal(pv.Values)
160+ }
161+ return []byte("null"), nil
162+}
163+
164+func rowCount(pvs []PlanValue, bindVars map[string]*querypb.BindVariable) (int, error) {
165+ count := -1
166+ setCount := func(l int) error {
167+ switch count {
168+ case -1:
169+ count = l
170+ return nil
171+ case l:
172+ return nil
173+ default:
174+ return errors.New("mismatch in number of column values")
175+ }
176+ }
177+
178+ for _, pv := range pvs {
179+ switch {
180+ case pv.Key != "" || !pv.Value.IsNull():
181+ continue
182+ case pv.Values != nil:
183+ if err := setCount(len(pv.Values)); err != nil {
184+ return 0, err
185+ }
186+ case pv.ListKey != "":
187+ bv, err := pv.lookupList(bindVars)
188+ if err != nil {
189+ return 0, err
190+ }
191+ if err := setCount(len(bv.Values)); err != nil {
192+ return 0, err
193+ }
194+ }
195+ }
196+
197+ if count == -1 {
198+ // If there were no lists inside, it was a single row.
199+ // Note that count can never be 0 because there is enough
200+ // protection at the top level: list bind vars must have
201+ // at least one value (enforced by vtgate), and AST lists
202+ // must have at least one value (enforced by the parser).
203+ // Also lists created internally after vtgate validation
204+ // ensure at least one value.
205+ // TODO(sougou): verify and change API to enforce this.
206+ return 1, nil
207+ }
208+ return count, nil
209+}
210+
211+// ResolveRows resolves a []PlanValue as rows based on the supplied bindvars.
212+func ResolveRows(pvs []PlanValue, bindVars map[string]*querypb.BindVariable) ([][]Value, error) {
213+ count, err := rowCount(pvs, bindVars)
214+ if err != nil {
215+ return nil, err
216+ }
217+
218+ // Allocate the rows.
219+ rows := make([][]Value, count)
220+ for i := range rows {
221+ rows[i] = make([]Value, len(pvs))
222+ }
223+
224+ // Using j becasue we're resolving by columns.
225+ for j, pv := range pvs {
226+ switch {
227+ case pv.Key != "":
228+ bv, err := pv.lookupValue(bindVars)
229+ if err != nil {
230+ return nil, err
231+ }
232+ for i := range rows {
233+ rows[i][j] = MakeTrusted(bv.Type, bv.Value)
234+ }
235+ case !pv.Value.IsNull():
236+ for i := range rows {
237+ rows[i][j] = pv.Value
238+ }
239+ case pv.ListKey != "":
240+ bv, err := pv.lookupList(bindVars)
241+ if err != nil {
242+ // This code is unreachable because pvRowCount already checks this.
243+ return nil, err
244+ }
245+ for i := range rows {
246+ rows[i][j] = MakeTrusted(bv.Values[i].Type, bv.Values[i].Value)
247+ }
248+ case pv.Values != nil:
249+ for i := range rows {
250+ rows[i][j], err = pv.Values[i].ResolveValue(bindVars)
251+ if err != nil {
252+ return nil, err
253+ }
254+ }
255+ // default case is a NULL value, which the row values are already initialized to.
256+ }
257+ }
258+ return rows, nil
259+}
vendor/github.com/xwb1989/sqlparser/dependency/sqltypes/testing.go+154−0
@@ -0,0 +1,154 @@
1+/*
2+Copyright 2017 Google Inc.
3+
4+Licensed under the Apache License, Version 2.0 (the "License");
5+you may not use this file except in compliance with the License.
6+You may obtain a copy of the License at
7+
8+ http://www.apache.org/licenses/LICENSE-2.0
9+
10+Unless required by applicable law or agreed to in writing, software
11+distributed under the License is distributed on an "AS IS" BASIS,
12+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+See the License for the specific language governing permissions and
14+limitations under the License.
15+*/
16+
17+package sqltypes
18+
19+import (
20+ querypb "github.com/xwb1989/sqlparser/dependency/querypb"
21+)
22+
23+// Functions in this file should only be used for testing.
24+// This is an experiment to see if test code bloat can be
25+// reduced and readability improved.
26+
27+/*
28+// MakeTestFields builds a []*querypb.Field for testing.
29+// fields := sqltypes.MakeTestFields(
30+// "a|b",
31+// "int64|varchar",
32+// )
33+// The field types are as defined in querypb and are case
34+// insensitive. Column delimiters must be used only to sepearate
35+// strings and not at the beginning or the end.
36+func MakeTestFields(names, types string) []*querypb.Field {
37+ n := split(names)
38+ t := split(types)
39+ var fields []*querypb.Field
40+ for i := range n {
41+ fields = append(fields, &querypb.Field{
42+ Name: n[i],
43+ Type: querypb.Type(querypb.Type_value[strings.ToUpper(t[i])]),
44+ })
45+ }
46+ return fields
47+}
48+
49+// MakeTestResult builds a *sqltypes.Result object for testing.
50+// result := sqltypes.MakeTestResult(
51+// fields,
52+// " 1|a",
53+// "10|abcd",
54+// )
55+// The field type values are set as the types for the rows built.
56+// Spaces are trimmed from row values. "null" is treated as NULL.
57+func MakeTestResult(fields []*querypb.Field, rows ...string) *Result {
58+ result := &Result{
59+ Fields: fields,
60+ }
61+ if len(rows) > 0 {
62+ result.Rows = make([][]Value, len(rows))
63+ }
64+ for i, row := range rows {
65+ result.Rows[i] = make([]Value, len(fields))
66+ for j, col := range split(row) {
67+ if col == "null" {
68+ continue
69+ }
70+ result.Rows[i][j] = MakeTrusted(fields[j].Type, []byte(col))
71+ }
72+ }
73+ result.RowsAffected = uint64(len(result.Rows))
74+ return result
75+}
76+
77+// MakeTestStreamingResults builds a list of results for streaming.
78+// results := sqltypes.MakeStreamingResults(
79+// fields,
80+// "1|a",
81+// "2|b",
82+// "---",
83+// "c|c",
84+// )
85+// The first result contains only the fields. Subsequent results
86+// are built using the field types. Every input that starts with a "-"
87+// is treated as streaming delimiter for one result. A final
88+// delimiter must not be supplied.
89+func MakeTestStreamingResults(fields []*querypb.Field, rows ...string) []*Result {
90+ var results []*Result
91+ results = append(results, &Result{Fields: fields})
92+ start := 0
93+ cur := 0
94+ // Add a final streaming delimiter to simplify the loop below.
95+ rows = append(rows, "-")
96+ for cur < len(rows) {
97+ if rows[cur][0] != '-' {
98+ cur++
99+ continue
100+ }
101+ result := MakeTestResult(fields, rows[start:cur]...)
102+ result.Fields = nil
103+ result.RowsAffected = 0
104+ results = append(results, result)
105+ start = cur + 1
106+ cur = start
107+ }
108+ return results
109+}
110+*/
111+
112+// TestBindVariable makes a *querypb.BindVariable from
113+// an interface{}.It panics on invalid input.
114+// This function should only be used for testing.
115+func TestBindVariable(v interface{}) *querypb.BindVariable {
116+ if v == nil {
117+ return NullBindVariable
118+ }
119+ bv, err := BuildBindVariable(v)
120+ if err != nil {
121+ panic(err)
122+ }
123+ return bv
124+}
125+
126+// TestValue builds a Value from typ and val.
127+// This function should only be used for testing.
128+func TestValue(typ querypb.Type, val string) Value {
129+ return MakeTrusted(typ, []byte(val))
130+}
131+
132+/*
133+// PrintResults prints []*Results into a string.
134+// This function should only be used for testing.
135+func PrintResults(results []*Result) string {
136+ b := new(bytes.Buffer)
137+ for i, r := range results {
138+ if i == 0 {
139+ fmt.Fprintf(b, "%v", r)
140+ continue
141+ }
142+ fmt.Fprintf(b, ", %v", r)
143+ }
144+ return b.String()
145+}
146+
147+func split(str string) []string {
148+ splits := strings.Split(str, "|")
149+ for i, v := range splits {
150+ splits[i] = strings.TrimSpace(v)
151+ }
152+ return splits
153+}
154+*/
vendor/github.com/xwb1989/sqlparser/dependency/sqltypes/type.go+288−0
@@ -0,0 +1,288 @@
1+/*
2+Copyright 2017 Google Inc.
3+
4+Licensed under the Apache License, Version 2.0 (the "License");
5+you may not use this file except in compliance with the License.
6+You may obtain a copy of the License at
7+
8+ http://www.apache.org/licenses/LICENSE-2.0
9+
10+Unless required by applicable law or agreed to in writing, software
11+distributed under the License is distributed on an "AS IS" BASIS,
12+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+See the License for the specific language governing permissions and
14+limitations under the License.
15+*/
16+
17+package sqltypes
18+
19+import (
20+ "fmt"
21+
22+ "github.com/xwb1989/sqlparser/dependency/querypb"
23+)
24+
25+// This file provides wrappers and support
26+// functions for querypb.Type.
27+
28+// These bit flags can be used to query on the
29+// common properties of types.
30+const (
31+ flagIsIntegral = int(querypb.Flag_ISINTEGRAL)
32+ flagIsUnsigned = int(querypb.Flag_ISUNSIGNED)
33+ flagIsFloat = int(querypb.Flag_ISFLOAT)
34+ flagIsQuoted = int(querypb.Flag_ISQUOTED)
35+ flagIsText = int(querypb.Flag_ISTEXT)
36+ flagIsBinary = int(querypb.Flag_ISBINARY)
37+)
38+
39+// IsIntegral returns true if querypb.Type is an integral
40+// (signed/unsigned) that can be represented using
41+// up to 64 binary bits.
42+// If you have a Value object, use its member function.
43+func IsIntegral(t querypb.Type) bool {
44+ return int(t)&flagIsIntegral == flagIsIntegral
45+}
46+
47+// IsSigned returns true if querypb.Type is a signed integral.
48+// If you have a Value object, use its member function.
49+func IsSigned(t querypb.Type) bool {
50+ return int(t)&(flagIsIntegral|flagIsUnsigned) == flagIsIntegral
51+}
52+
53+// IsUnsigned returns true if querypb.Type is an unsigned integral.
54+// Caution: this is not the same as !IsSigned.
55+// If you have a Value object, use its member function.
56+func IsUnsigned(t querypb.Type) bool {
57+ return int(t)&(flagIsIntegral|flagIsUnsigned) == flagIsIntegral|flagIsUnsigned
58+}
59+
60+// IsFloat returns true is querypb.Type is a floating point.
61+// If you have a Value object, use its member function.
62+func IsFloat(t querypb.Type) bool {
63+ return int(t)&flagIsFloat == flagIsFloat
64+}
65+
66+// IsQuoted returns true if querypb.Type is a quoted text or binary.
67+// If you have a Value object, use its member function.
68+func IsQuoted(t querypb.Type) bool {
69+ return int(t)&flagIsQuoted == flagIsQuoted
70+}
71+
72+// IsText returns true if querypb.Type is a text.
73+// If you have a Value object, use its member function.
74+func IsText(t querypb.Type) bool {
75+ return int(t)&flagIsText == flagIsText
76+}
77+
78+// IsBinary returns true if querypb.Type is a binary.
79+// If you have a Value object, use its member function.
80+func IsBinary(t querypb.Type) bool {
81+ return int(t)&flagIsBinary == flagIsBinary
82+}
83+
84+// isNumber returns true if the type is any type of number.
85+func isNumber(t querypb.Type) bool {
86+ return IsIntegral(t) || IsFloat(t) || t == Decimal
87+}
88+
89+// Vitess data types. These are idiomatically
90+// named synonyms for the querypb.Type values.
91+// Although these constants are interchangeable,
92+// they should be treated as different from querypb.Type.
93+// Use the synonyms only to refer to the type in Value.
94+// For proto variables, use the querypb.Type constants
95+// instead.
96+// The following conditions are non-overlapping
97+// and cover all types: IsSigned(), IsUnsigned(),
98+// IsFloat(), IsQuoted(), Null, Decimal, Expression.
99+// Also, IsIntegral() == (IsSigned()||IsUnsigned()).
100+// TestCategory needs to be updated accordingly if
101+// you add a new type.
102+// If IsBinary or IsText is true, then IsQuoted is
103+// also true. But there are IsQuoted types that are
104+// neither binary or text.
105+// querypb.Type_TUPLE is not included in this list
106+// because it's not a valid Value type.
107+// TODO(sougou): provide a categorization function
108+// that returns enums, which will allow for cleaner
109+// switch statements for those who want to cover types
110+// by their category.
111+const (
112+ Null = querypb.Type_NULL_TYPE
113+ Int8 = querypb.Type_INT8
114+ Uint8 = querypb.Type_UINT8
115+ Int16 = querypb.Type_INT16
116+ Uint16 = querypb.Type_UINT16
117+ Int24 = querypb.Type_INT24
118+ Uint24 = querypb.Type_UINT24
119+ Int32 = querypb.Type_INT32
120+ Uint32 = querypb.Type_UINT32
121+ Int64 = querypb.Type_INT64
122+ Uint64 = querypb.Type_UINT64
123+ Float32 = querypb.Type_FLOAT32
124+ Float64 = querypb.Type_FLOAT64
125+ Timestamp = querypb.Type_TIMESTAMP
126+ Date = querypb.Type_DATE
127+ Time = querypb.Type_TIME
128+ Datetime = querypb.Type_DATETIME
129+ Year = querypb.Type_YEAR
130+ Decimal = querypb.Type_DECIMAL
131+ Text = querypb.Type_TEXT
132+ Blob = querypb.Type_BLOB
133+ VarChar = querypb.Type_VARCHAR
134+ VarBinary = querypb.Type_VARBINARY
135+ Char = querypb.Type_CHAR
136+ Binary = querypb.Type_BINARY
137+ Bit = querypb.Type_BIT
138+ Enum = querypb.Type_ENUM
139+ Set = querypb.Type_SET
140+ Geometry = querypb.Type_GEOMETRY
141+ TypeJSON = querypb.Type_JSON
142+ Expression = querypb.Type_EXPRESSION
143+)
144+
145+// bit-shift the mysql flags by two byte so we
146+// can merge them with the mysql or vitess types.
147+const (
148+ mysqlUnsigned = 32
149+ mysqlBinary = 128
150+ mysqlEnum = 256
151+ mysqlSet = 2048
152+)
153+
154+// If you add to this map, make sure you add a test case
155+// in tabletserver/endtoend.
156+var mysqlToType = map[int64]querypb.Type{
157+ 1: Int8,
158+ 2: Int16,
159+ 3: Int32,
160+ 4: Float32,
161+ 5: Float64,
162+ 6: Null,
163+ 7: Timestamp,
164+ 8: Int64,
165+ 9: Int24,
166+ 10: Date,
167+ 11: Time,
168+ 12: Datetime,
169+ 13: Year,
170+ 16: Bit,
171+ 245: TypeJSON,
172+ 246: Decimal,
173+ 249: Text,
174+ 250: Text,
175+ 251: Text,
176+ 252: Text,
177+ 253: VarChar,
178+ 254: Char,
179+ 255: Geometry,
180+}
181+
182+// modifyType modifies the vitess type based on the
183+// mysql flag. The function checks specific flags based
184+// on the type. This allows us to ignore stray flags
185+// that MySQL occasionally sets.
186+func modifyType(typ querypb.Type, flags int64) querypb.Type {
187+ switch typ {
188+ case Int8:
189+ if flags&mysqlUnsigned != 0 {
190+ return Uint8
191+ }
192+ return Int8
193+ case Int16:
194+ if flags&mysqlUnsigned != 0 {
195+ return Uint16
196+ }
197+ return Int16
198+ case Int32:
199+ if flags&mysqlUnsigned != 0 {
200+ return Uint32
201+ }
202+ return Int32
203+ case Int64:
204+ if flags&mysqlUnsigned != 0 {
205+ return Uint64
206+ }
207+ return Int64
208+ case Int24:
209+ if flags&mysqlUnsigned != 0 {
210+ return Uint24
211+ }
212+ return Int24
213+ case Text:
214+ if flags&mysqlBinary != 0 {
215+ return Blob
216+ }
217+ return Text
218+ case VarChar:
219+ if flags&mysqlBinary != 0 {
220+ return VarBinary
221+ }
222+ return VarChar
223+ case Char:
224+ if flags&mysqlBinary != 0 {
225+ return Binary
226+ }
227+ if flags&mysqlEnum != 0 {
228+ return Enum
229+ }
230+ if flags&mysqlSet != 0 {
231+ return Set
232+ }
233+ return Char
234+ }
235+ return typ
236+}
237+
238+// MySQLToType computes the vitess type from mysql type and flags.
239+func MySQLToType(mysqlType, flags int64) (typ querypb.Type, err error) {
240+ result, ok := mysqlToType[mysqlType]
241+ if !ok {
242+ return 0, fmt.Errorf("unsupported type: %d", mysqlType)
243+ }
244+ return modifyType(result, flags), nil
245+}
246+
247+// typeToMySQL is the reverse of mysqlToType.
248+var typeToMySQL = map[querypb.Type]struct {
249+ typ int64
250+ flags int64
251+}{
252+ Int8: {typ: 1},
253+ Uint8: {typ: 1, flags: mysqlUnsigned},
254+ Int16: {typ: 2},
255+ Uint16: {typ: 2, flags: mysqlUnsigned},
256+ Int32: {typ: 3},
257+ Uint32: {typ: 3, flags: mysqlUnsigned},
258+ Float32: {typ: 4},
259+ Float64: {typ: 5},
260+ Null: {typ: 6, flags: mysqlBinary},
261+ Timestamp: {typ: 7},
262+ Int64: {typ: 8},
263+ Uint64: {typ: 8, flags: mysqlUnsigned},
264+ Int24: {typ: 9},
265+ Uint24: {typ: 9, flags: mysqlUnsigned},
266+ Date: {typ: 10, flags: mysqlBinary},
267+ Time: {typ: 11, flags: mysqlBinary},
268+ Datetime: {typ: 12, flags: mysqlBinary},
269+ Year: {typ: 13, flags: mysqlUnsigned},
270+ Bit: {typ: 16, flags: mysqlUnsigned},
271+ TypeJSON: {typ: 245},
272+ Decimal: {typ: 246},
273+ Text: {typ: 252},
274+ Blob: {typ: 252, flags: mysqlBinary},
275+ VarChar: {typ: 253},
276+ VarBinary: {typ: 253, flags: mysqlBinary},
277+ Char: {typ: 254},
278+ Binary: {typ: 254, flags: mysqlBinary},
279+ Enum: {typ: 254, flags: mysqlEnum},
280+ Set: {typ: 254, flags: mysqlSet},
281+ Geometry: {typ: 255},
282+}
283+
284+// TypeToMySQL returns the equivalent mysql type and flag for a vitess type.
285+func TypeToMySQL(typ querypb.Type) (mysqlType, flags int64) {
286+ val := typeToMySQL[typ]
287+ return val.typ, val.flags
288+}
vendor/github.com/xwb1989/sqlparser/dependency/sqltypes/value.go+376−0
@@ -0,0 +1,376 @@
1+/*
2+Copyright 2017 Google Inc.
3+
4+Licensed under the Apache License, Version 2.0 (the "License");
5+you may not use this file except in compliance with the License.
6+You may obtain a copy of the License at
7+
8+ http://www.apache.org/licenses/LICENSE-2.0
9+
10+Unless required by applicable law or agreed to in writing, software
11+distributed under the License is distributed on an "AS IS" BASIS,
12+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+See the License for the specific language governing permissions and
14+limitations under the License.
15+*/
16+
17+// Package sqltypes implements interfaces and types that represent SQL values.
18+package sqltypes
19+
20+import (
21+ "encoding/base64"
22+ "encoding/json"
23+ "fmt"
24+ "strconv"
25+
26+ "github.com/xwb1989/sqlparser/dependency/bytes2"
27+ "github.com/xwb1989/sqlparser/dependency/hack"
28+
29+ "github.com/xwb1989/sqlparser/dependency/querypb"
30+)
31+
32+var (
33+ // NULL represents the NULL value.
34+ NULL = Value{}
35+
36+ // DontEscape tells you if a character should not be escaped.
37+ DontEscape = byte(255)
38+
39+ nullstr = []byte("null")
40+)
41+
42+// BinWriter interface is used for encoding values.
43+// Types like bytes.Buffer conform to this interface.
44+// We expect the writer objects to be in-memory buffers.
45+// So, we don't expect the write operations to fail.
46+type BinWriter interface {
47+ Write([]byte) (int, error)
48+}
49+
50+// Value can store any SQL value. If the value represents
51+// an integral type, the bytes are always stored as a cannonical
52+// representation that matches how MySQL returns such values.
53+type Value struct {
54+ typ querypb.Type
55+ val []byte
56+}
57+
58+// NewValue builds a Value using typ and val. If the value and typ
59+// don't match, it returns an error.
60+func NewValue(typ querypb.Type, val []byte) (v Value, err error) {
61+ switch {
62+ case IsSigned(typ):
63+ if _, err := strconv.ParseInt(string(val), 0, 64); err != nil {
64+ return NULL, err
65+ }
66+ return MakeTrusted(typ, val), nil
67+ case IsUnsigned(typ):
68+ if _, err := strconv.ParseUint(string(val), 0, 64); err != nil {
69+ return NULL, err
70+ }
71+ return MakeTrusted(typ, val), nil
72+ case IsFloat(typ) || typ == Decimal:
73+ if _, err := strconv.ParseFloat(string(val), 64); err != nil {
74+ return NULL, err
75+ }
76+ return MakeTrusted(typ, val), nil
77+ case IsQuoted(typ) || typ == Null:
78+ return MakeTrusted(typ, val), nil
79+ }
80+ // All other types are unsafe or invalid.
81+ return NULL, fmt.Errorf("invalid type specified for MakeValue: %v", typ)
82+}
83+
84+// MakeTrusted makes a new Value based on the type.
85+// This function should only be used if you know the value
86+// and type conform to the rules. Every place this function is
87+// called, a comment is needed that explains why it's justified.
88+// Exceptions: The current package and mysql package do not need
89+// comments. Other packages can also use the function to create
90+// VarBinary or VarChar values.
91+func MakeTrusted(typ querypb.Type, val []byte) Value {
92+ if typ == Null {
93+ return NULL
94+ }
95+ return Value{typ: typ, val: val}
96+}
97+
98+// NewInt64 builds an Int64 Value.
99+func NewInt64(v int64) Value {
100+ return MakeTrusted(Int64, strconv.AppendInt(nil, v, 10))
101+}
102+
103+// NewInt32 builds an Int64 Value.
104+func NewInt32(v int32) Value {
105+ return MakeTrusted(Int32, strconv.AppendInt(nil, int64(v), 10))
106+}
107+
108+// NewUint64 builds an Uint64 Value.
109+func NewUint64(v uint64) Value {
110+ return MakeTrusted(Uint64, strconv.AppendUint(nil, v, 10))
111+}
112+
113+// NewFloat64 builds an Float64 Value.
114+func NewFloat64(v float64) Value {
115+ return MakeTrusted(Float64, strconv.AppendFloat(nil, v, 'g', -1, 64))
116+}
117+
118+// NewVarChar builds a VarChar Value.
119+func NewVarChar(v string) Value {
120+ return MakeTrusted(VarChar, []byte(v))
121+}
122+
123+// NewVarBinary builds a VarBinary Value.
124+// The input is a string because it's the most common use case.
125+func NewVarBinary(v string) Value {
126+ return MakeTrusted(VarBinary, []byte(v))
127+}
128+
129+// NewIntegral builds an integral type from a string representaion.
130+// The type will be Int64 or Uint64. Int64 will be preferred where possible.
131+func NewIntegral(val string) (n Value, err error) {
132+ signed, err := strconv.ParseInt(val, 0, 64)
133+ if err == nil {
134+ return MakeTrusted(Int64, strconv.AppendInt(nil, signed, 10)), nil
135+ }
136+ unsigned, err := strconv.ParseUint(val, 0, 64)
137+ if err != nil {
138+ return Value{}, err
139+ }
140+ return MakeTrusted(Uint64, strconv.AppendUint(nil, unsigned, 10)), nil
141+}
142+
143+// InterfaceToValue builds a value from a go type.
144+// Supported types are nil, int64, uint64, float64,
145+// string and []byte.
146+// This function is deprecated. Use the type-specific
147+// functions instead.
148+func InterfaceToValue(goval interface{}) (Value, error) {
149+ switch goval := goval.(type) {
150+ case nil:
151+ return NULL, nil
152+ case []byte:
153+ return MakeTrusted(VarBinary, goval), nil
154+ case int64:
155+ return NewInt64(goval), nil
156+ case uint64:
157+ return NewUint64(goval), nil
158+ case float64:
159+ return NewFloat64(goval), nil
160+ case string:
161+ return NewVarChar(goval), nil
162+ default:
163+ return NULL, fmt.Errorf("unexpected type %T: %v", goval, goval)
164+ }
165+}
166+
167+// Type returns the type of Value.
168+func (v Value) Type() querypb.Type {
169+ return v.typ
170+}
171+
172+// Raw returns the internal represenation of the value. For newer types,
173+// this may not match MySQL's representation.
174+func (v Value) Raw() []byte {
175+ return v.val
176+}
177+
178+// ToBytes returns the value as MySQL would return it as []byte.
179+// In contrast, Raw returns the internal representation of the Value, which may not
180+// match MySQL's representation for newer types.
181+// If the value is not convertible like in the case of Expression, it returns nil.
182+func (v Value) ToBytes() []byte {
183+ if v.typ == Expression {
184+ return nil
185+ }
186+ return v.val
187+}
188+
189+// Len returns the length.
190+func (v Value) Len() int {
191+ return len(v.val)
192+}
193+
194+// ToString returns the value as MySQL would return it as string.
195+// If the value is not convertible like in the case of Expression, it returns nil.
196+func (v Value) ToString() string {
197+ if v.typ == Expression {
198+ return ""
199+ }
200+ return hack.String(v.val)
201+}
202+
203+// String returns a printable version of the value.
204+func (v Value) String() string {
205+ if v.typ == Null {
206+ return "NULL"
207+ }
208+ if v.IsQuoted() {
209+ return fmt.Sprintf("%v(%q)", v.typ, v.val)
210+ }
211+ return fmt.Sprintf("%v(%s)", v.typ, v.val)
212+}
213+
214+// EncodeSQL encodes the value into an SQL statement. Can be binary.
215+func (v Value) EncodeSQL(b BinWriter) {
216+ switch {
217+ case v.typ == Null:
218+ b.Write(nullstr)
219+ case v.IsQuoted():
220+ encodeBytesSQL(v.val, b)
221+ default:
222+ b.Write(v.val)
223+ }
224+}
225+
226+// EncodeASCII encodes the value using 7-bit clean ascii bytes.
227+func (v Value) EncodeASCII(b BinWriter) {
228+ switch {
229+ case v.typ == Null:
230+ b.Write(nullstr)
231+ case v.IsQuoted():
232+ encodeBytesASCII(v.val, b)
233+ default:
234+ b.Write(v.val)
235+ }
236+}
237+
238+// IsNull returns true if Value is null.
239+func (v Value) IsNull() bool {
240+ return v.typ == Null
241+}
242+
243+// IsIntegral returns true if Value is an integral.
244+func (v Value) IsIntegral() bool {
245+ return IsIntegral(v.typ)
246+}
247+
248+// IsSigned returns true if Value is a signed integral.
249+func (v Value) IsSigned() bool {
250+ return IsSigned(v.typ)
251+}
252+
253+// IsUnsigned returns true if Value is an unsigned integral.
254+func (v Value) IsUnsigned() bool {
255+ return IsUnsigned(v.typ)
256+}
257+
258+// IsFloat returns true if Value is a float.
259+func (v Value) IsFloat() bool {
260+ return IsFloat(v.typ)
261+}
262+
263+// IsQuoted returns true if Value must be SQL-quoted.
264+func (v Value) IsQuoted() bool {
265+ return IsQuoted(v.typ)
266+}
267+
268+// IsText returns true if Value is a collatable text.
269+func (v Value) IsText() bool {
270+ return IsText(v.typ)
271+}
272+
273+// IsBinary returns true if Value is binary.
274+func (v Value) IsBinary() bool {
275+ return IsBinary(v.typ)
276+}
277+
278+// MarshalJSON should only be used for testing.
279+// It's not a complete implementation.
280+func (v Value) MarshalJSON() ([]byte, error) {
281+ switch {
282+ case v.IsQuoted():
283+ return json.Marshal(v.ToString())
284+ case v.typ == Null:
285+ return nullstr, nil
286+ }
287+ return v.val, nil
288+}
289+
290+// UnmarshalJSON should only be used for testing.
291+// It's not a complete implementation.
292+func (v *Value) UnmarshalJSON(b []byte) error {
293+ if len(b) == 0 {
294+ return fmt.Errorf("error unmarshaling empty bytes")
295+ }
296+ var val interface{}
297+ var err error
298+ switch b[0] {
299+ case '-':
300+ var ival int64
301+ err = json.Unmarshal(b, &ival)
302+ val = ival
303+ case '"':
304+ var bval []byte
305+ err = json.Unmarshal(b, &bval)
306+ val = bval
307+ case 'n': // null
308+ err = json.Unmarshal(b, &val)
309+ default:
310+ var uval uint64
311+ err = json.Unmarshal(b, &uval)
312+ val = uval

64 more lines aren’t shown here.

vendor/github.com/xwb1989/sqlparser/encodable.go+99−0

This diff is large: its 100 lines aren’t shown here.

vendor/github.com/xwb1989/sqlparser/impossible_query.go+39−0

This diff is large: its 40 lines aren’t shown here.

vendor/github.com/xwb1989/sqlparser/normalizer.go+224−0

This diff is large: its 225 lines aren’t shown here.

vendor/github.com/xwb1989/sqlparser/parsed_query.go+119−0

This diff is large: its 120 lines aren’t shown here.

vendor/github.com/xwb1989/sqlparser/redact_query.go+19−0

This diff is large: its 20 lines aren’t shown here.

vendor/github.com/xwb1989/sqlparser/sql.go+6136−0

This diff is large: its 6137 lines aren’t shown here.

vendor/github.com/xwb1989/sqlparser/sql.y+3159−0

This diff is large: its 3160 lines aren’t shown here.

vendor/github.com/xwb1989/sqlparser/token.go+950−0

This diff is large: its 951 lines aren’t shown here.

vendor/github.com/xwb1989/sqlparser/tracked_buffer.go+140−0

This diff is large: its 141 lines aren’t shown here.

vendor/vendor.json+48−0

This diff is large: its 69 lines aren’t shown here.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t