diff --git a/Gopkg.lock b/Gopkg.lock index 0c21b385f..3d49e48f6 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -485,7 +485,7 @@ "sql/parse", "sql/plan" ] - revision = "dcc1c537eb7e2bc20ce69d03732e8aa8feaa8612" + revision = "c062e48a1a1b718e43fecdbc0ec350d97daff90c" [[projects]] name = "gopkg.in/src-d/go-siva.v1" @@ -538,6 +538,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "cefe7eedc3e61db6656997a3fd8892cccbb3351193b8c36cbc922901d47d44a4" + inputs-digest = "700e638275a2b54d2706ab58ccbbae5d309b130480ff8b9799f6070cb5034a98" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 30df6520c..ed71d3d5d 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -1,6 +1,6 @@ [[constraint]] name = "gopkg.in/src-d/go-mysql-server.v0" - revision = "dcc1c537eb7e2bc20ce69d03732e8aa8feaa8612" + revision = "c062e48a1a1b718e43fecdbc0ec350d97daff90c" [[constraint]] name = "github.com/jessevdk/go-flags" diff --git a/docs/using-gitbase/functions.md b/docs/using-gitbase/functions.md index 79287d839..03b242aa3 100644 --- a/docs/using-gitbase/functions.md +++ b/docs/using-gitbase/functions.md @@ -14,4 +14,4 @@ To make some common tasks easier for the user, there are some functions to inter ## Standard functions -You can check standard functions in [`go-mysql-server` documentation](https://github.com/src-d/go-mysql-server/tree/dcc1c537eb7e2bc20ce69d03732e8aa8feaa8612#custom-functions). +You can check standard functions in [`go-mysql-server` documentation](https://github.com/src-d/go-mysql-server/tree/c062e48a1a1b718e43fecdbc0ec350d97daff90c#custom-functions). diff --git a/docs/using-gitbase/supported-syntax.md b/docs/using-gitbase/supported-syntax.md index fbf4b8503..0cd7182ad 100644 --- a/docs/using-gitbase/supported-syntax.md +++ b/docs/using-gitbase/supported-syntax.md @@ -1,3 +1,3 @@ ## Supported syntax -To see the SQL subset currently supported take a look at [this list](https://github.com/src-d/go-mysql-server/blob/dcc1c537eb7e2bc20ce69d03732e8aa8feaa8612/SUPPORTED.md) from [src-d/go-mysql-server](https://github.com/src-d/go-mysql-server). +To see the SQL subset currently supported take a look at [this list](https://github.com/src-d/go-mysql-server/blob/c062e48a1a1b718e43fecdbc0ec350d97daff90c/SUPPORTED.md) from [src-d/go-mysql-server](https://github.com/src-d/go-mysql-server). diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/README.md b/vendor/gopkg.in/src-d/go-mysql-server.v0/README.md index c05a77c4b..d80607be3 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/README.md +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/README.md @@ -51,12 +51,12 @@ We are continuously adding more functionality to go-mysql-server. We support a s ## Custom functions -- `IS_BINARY(blob)`: returns whether a BLOB is a binary file or not. -- `SUBSTRING(str,pos)`, ` SUBSTRING(str,pos,len)`: return a substring from the provided string. +- `IS_BINARY(blob)`: Returns whether a BLOB is a binary file or not. +- `SUBSTRING(str,pos)`, `SUBSTRING(str,pos,len)`: Return a substring from the provided string. - Date and Timestamp functions: `YEAR(date)`, `MONTH(date)`, `DAY(date)`, `HOUR(date)`, `MINUTE(date)`, `SECOND(date)`, `DAYOFYEAR(date)`. - `ARRAY_LENGTH(json)`: If the json representation is an array, this function returns its size. -- `SPLIT(str,sep)`: receives a string and a delimiter and returns the parts of the string splitted by the delimiter as a JSON array of strings. - +- `SPLIT(str,sep)`: Receives a string and a separator and returns the parts of the string split by the separator as a JSON array of strings. +- `CONCAT(...)`: Concatenate any group of fields into a single string. ## Example `go-mysql-server` contains a SQL engine and server implementation. So, if you want to start a server, first instantiate the engine and pass your `sql.Database` implementation. diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/engine.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/engine.go index 3ba23145f..b1ce100bf 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/engine.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/engine.go @@ -16,6 +16,7 @@ type Engine struct { // New creates a new Engine func New(c *sql.Catalog, a *analyzer.Analyzer) *Engine { + c.RegisterFunctions(function.Defaults) return &Engine{c, a} } diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/engine_test.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/engine_test.go index 398b17bb8..207a3444a 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/engine_test.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/engine_test.go @@ -276,6 +276,20 @@ var queries = []struct { {int32(1), int64(1)}, }, }, + { + `SELECT CONCAT("a", "b", "c")`, + []sql.Row{ + {string("abc")}, + }, + }, + { + "SELECT concat(s, i) FROM mytable", + []sql.Row{ + {string("first row1")}, + {string("second row2")}, + {string("third row3")}, + }, + }, } func TestQueries(t *testing.T) { @@ -827,9 +841,6 @@ func TestTracing(t *testing.T) { "plan.Project", "plan.Filter", "plan.PushdownProjectionAndFiltersTable", - "expression.Equals", - "expression.Equals", - "expression.Equals", } var spanOperations []string diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/analyzer/analyzer.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/analyzer/analyzer.go index eacc7bc37..92b686499 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/analyzer/analyzer.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/analyzer/analyzer.go @@ -3,6 +3,7 @@ package analyzer // import "gopkg.in/src-d/go-mysql-server.v0/sql/analyzer" import ( "os" + opentracing "github.com/opentracing/opentracing-go" "github.com/sirupsen/logrus" "gopkg.in/src-d/go-errors.v1" "gopkg.in/src-d/go-mysql-server.v0/sql" @@ -142,8 +143,9 @@ func (a *Analyzer) Log(msg string, args ...interface{}) { // Analyze the node and all its children. func (a *Analyzer) Analyze(ctx *sql.Context, n sql.Node) (sql.Node, error) { - span, ctx := ctx.Span("analyze") - span.LogKV("plan", n.String()) + span, ctx := ctx.Span("analyze", opentracing.Tags{ + "plan": n.String(), + }) prev := n var err error diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/arithmetic.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/arithmetic.go index 38214f755..b4a058a65 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/arithmetic.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/arithmetic.go @@ -130,9 +130,6 @@ func (a *Arithmetic) TransformUp(f sql.TransformExprFunc) (sql.Expression, error // Eval implements the Expression interface. func (a *Arithmetic) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { - span, ctx := ctx.Span("expression.(" + a.op + ")") - defer span.Finish() - lval, rval, err := a.evalLeftRight(ctx, row) if err != nil { return nil, err diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/between.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/between.go index aac2c1284..9853eb83e 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/between.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/between.go @@ -42,9 +42,6 @@ func (b *Between) Resolved() bool { // Eval implements the Expression interface. func (b *Between) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { - span, ctx := ctx.Span("expression.Between") - defer span.Finish() - typ := b.Val.Type() val, err := b.Val.Eval(ctx, row) if err != nil { diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/boolean.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/boolean.go index 726eb6c7b..5030f8ac4 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/boolean.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/boolean.go @@ -23,9 +23,6 @@ func (e *Not) Type() sql.Type { // Eval implements the Expression interface. func (e *Not) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { - span, ctx := ctx.Span("expression.Not") - defer span.Finish() - v, err := e.Child.Eval(ctx, row) if err != nil { return nil, err diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/comparison.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/comparison.go index c41bf3a25..e8ebc35cc 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/comparison.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/comparison.go @@ -144,9 +144,6 @@ func NewEquals(left sql.Expression, right sql.Expression) *Equals { // Eval implements the Expression interface. func (e *Equals) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { - span, ctx := ctx.Span("expression.Equals") - defer span.Finish() - result, err := e.Compare(ctx, row) if err != nil { if ErrNilOperand.Is(err) { @@ -190,9 +187,6 @@ func NewRegexp(left sql.Expression, right sql.Expression) *Regexp { // Eval implements the Expression interface. func (re *Regexp) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { - span, ctx := ctx.Span("expression.Regexp") - defer span.Finish() - if sql.IsText(re.Left().Type()) && sql.IsText(re.Right().Type()) { return re.compareRegexp(ctx, row) } @@ -268,9 +262,6 @@ func NewGreaterThan(left sql.Expression, right sql.Expression) *GreaterThan { // Eval implements the Expression interface. func (gt *GreaterThan) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { - span, ctx := ctx.Span("expression.GreaterThan") - defer span.Finish() - result, err := gt.Compare(ctx, row) if err != nil { if ErrNilOperand.Is(err) { @@ -314,9 +305,6 @@ func NewLessThan(left sql.Expression, right sql.Expression) *LessThan { // Eval implements the expression interface. func (lt *LessThan) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { - span, ctx := ctx.Span("expression.LessThan") - defer span.Finish() - result, err := lt.Compare(ctx, row) if err != nil { if ErrNilOperand.Is(err) { @@ -361,9 +349,6 @@ func NewGreaterThanOrEqual(left sql.Expression, right sql.Expression) *GreaterTh // Eval implements the Expression interface. func (gte *GreaterThanOrEqual) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { - span, ctx := ctx.Span("expression.GreaterThanOrEqual") - defer span.Finish() - result, err := gte.Compare(ctx, row) if err != nil { if ErrNilOperand.Is(err) { @@ -408,9 +393,6 @@ func NewLessThanOrEqual(left sql.Expression, right sql.Expression) *LessThanOrEq // Eval implements the Expression interface. func (lte *LessThanOrEqual) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { - span, ctx := ctx.Span("expression.LessThanOrEqual") - defer span.Finish() - result, err := lte.Compare(ctx, row) if err != nil { if ErrNilOperand.Is(err) { @@ -463,9 +445,6 @@ func NewIn(left sql.Expression, right sql.Expression) *In { // Eval implements the Expression interface. func (in *In) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { - span, ctx := ctx.Span("expression.In") - defer span.Finish() - typ := in.Left().Type() leftElems := sql.NumColumns(typ) left, err := in.Left().Eval(ctx, row) @@ -554,9 +533,6 @@ func NewNotIn(left sql.Expression, right sql.Expression) *NotIn { // Eval implements the Expression interface. func (in *NotIn) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { - span, ctx := ctx.Span("expression.NotIn") - defer span.Finish() - typ := in.Left().Type() leftElems := sql.NumColumns(typ) left, err := in.Left().Eval(ctx, row) diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/convert.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/convert.go index cafcf4e1a..28012d5b4 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/convert.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/convert.go @@ -7,7 +7,6 @@ import ( "strings" "time" - opentracing "github.com/opentracing/opentracing-go" "github.com/spf13/cast" errors "gopkg.in/src-d/go-errors.v1" "gopkg.in/src-d/go-mysql-server.v0/sql" @@ -91,9 +90,6 @@ func (c *Convert) TransformUp(f sql.TransformExprFunc) (sql.Expression, error) { // Eval implements the Expression interface. func (c *Convert) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { - span, ctx := ctx.Span("expression.Convert", opentracing.Tag{Key: "type", Value: c.castToType}) - defer span.Finish() - val, err := c.Child.Eval(ctx, row) if err != nil { return nil, err diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/aggregation/avg.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/aggregation/avg.go index d9f9a1465..ddc4ef2cf 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/aggregation/avg.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/aggregation/avg.go @@ -39,9 +39,6 @@ func (a *Avg) IsNullable() bool { // Eval implements AggregationExpression interface. (AggregationExpression[Expression]]) func (a *Avg) Eval(ctx *sql.Context, buffer sql.Row) (interface{}, error) { - span, ctx := ctx.Span("aggregation.Avg_Eval") - defer span.Finish() - isNoNum := buffer[2].(bool) if isNoNum { return float64(0), nil @@ -53,7 +50,6 @@ func (a *Avg) Eval(ctx *sql.Context, buffer sql.Row) (interface{}, error) { } avg := buffer[0] - span.LogKV("avg", avg) return avg, nil } diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/aggregation/count.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/aggregation/count.go index 8e5964de4..f28e30142 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/aggregation/count.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/aggregation/count.go @@ -85,10 +85,6 @@ func (c *Count) Merge(ctx *sql.Context, buffer, partial sql.Row) error { // Eval implements the Aggregation interface. func (c *Count) Eval(ctx *sql.Context, buffer sql.Row) (interface{}, error) { - span, ctx := ctx.Span("aggregation.Count_Eval") count := buffer[0] - span.LogKV("count", count) - span.Finish() - return count, nil } diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/aggregation/max.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/aggregation/max.go index 9f2cee986..1c793a8e1 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/aggregation/max.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/aggregation/max.go @@ -85,10 +85,6 @@ func (m *Max) Merge(ctx *sql.Context, buffer, partial sql.Row) error { // Eval implements the Aggregation interface. func (m *Max) Eval(ctx *sql.Context, buffer sql.Row) (interface{}, error) { - span, ctx := ctx.Span("aggregation.Max_Eval") max := buffer[0] - span.LogKV("max", max) - span.Finish() - return max, nil } diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/aggregation/min.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/aggregation/min.go index dc673a504..3fef114ea 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/aggregation/min.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/aggregation/min.go @@ -85,10 +85,6 @@ func (m *Min) Merge(ctx *sql.Context, buffer, partial sql.Row) error { // Eval implements the Aggregation interface func (m *Min) Eval(ctx *sql.Context, buffer sql.Row) (interface{}, error) { - span, ctx := ctx.Span("aggregation.Min_Eval") min := buffer[0] - span.LogKV("min", min) - span.Finish() - return min, nil } diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/aggregation/sum.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/aggregation/sum.go index fc7b6ca1b..104de369b 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/aggregation/sum.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/aggregation/sum.go @@ -73,10 +73,7 @@ func (m *Sum) Merge(ctx *sql.Context, buffer, partial sql.Row) error { // Eval implements the Aggregation interface. func (m *Sum) Eval(ctx *sql.Context, buffer sql.Row) (interface{}, error) { - span, ctx := ctx.Span("aggregation.Sum_Eval") sum := buffer[0] - span.LogKV("sum", sum) - span.Finish() return sum, nil } diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/arraylength.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/arraylength.go index f33dd2f4b..cfbc1cacc 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/arraylength.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/arraylength.go @@ -38,9 +38,6 @@ func (f *ArrayLength) TransformUp(fn sql.TransformExprFunc) (sql.Expression, err // Eval implements the Expression interface. func (f *ArrayLength) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { - span, ctx := ctx.Span("function.ArrayLength") - defer span.Finish() - if !sql.IsArray(f.Child.Type()) { return nil, sql.ErrInvalidType.New(f.Child.Type().Type().String()) } diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/isbinary.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/isbinary.go index 499b2618d..09f6dd3d0 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/isbinary.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/isbinary.go @@ -3,10 +3,6 @@ package function import ( "bytes" "fmt" - "time" - - "github.com/opentracing/opentracing-go" - "github.com/opentracing/opentracing-go/log" "gopkg.in/src-d/go-mysql-server.v0/sql" "gopkg.in/src-d/go-mysql-server.v0/sql/expression" @@ -27,21 +23,6 @@ func (ib *IsBinary) Eval( ctx *sql.Context, row sql.Row, ) (interface{}, error) { - var blobSize int - span, ctx := ctx.Span("function.IsBinary") - defer func() { - span.FinishWithOptions(opentracing.FinishOptions{ - LogRecords: []opentracing.LogRecord{ - { - Timestamp: time.Now(), - Fields: []log.Field{ - log.Int("blobsize", blobSize), - }, - }, - }, - }) - }() - v, err := ib.Child.Eval(ctx, row) if err != nil { return nil, err @@ -57,7 +38,6 @@ func (ib *IsBinary) Eval( } blobBytes := blob.([]byte) - blobSize = len(blobBytes) return isBinary(blobBytes), nil } diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/registry.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/registry.go index a18ae0e2a..007d955ef 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/registry.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/registry.go @@ -33,4 +33,5 @@ var Defaults = sql.Functions{ "dayofyear": sql.Function1(NewDayOfYear), "array_length": sql.Function1(NewArrayLength), "split": sql.Function2(NewSplit), + "concat": sql.FunctionN(NewConcat), } diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/substring.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/substring.go index a9ddfc7c7..b40321ce7 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/substring.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/substring.go @@ -50,9 +50,6 @@ func (s *Substring) Eval( ctx *sql.Context, row sql.Row, ) (interface{}, error) { - span, ctx := ctx.Span("function.Substring") - defer span.Finish() - str, err := s.str.Eval(ctx, row) if err != nil { return nil, err diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/time.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/time.go index 315dbf235..220d02274 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/time.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/function/time.go @@ -51,8 +51,6 @@ func (y *Year) Type() sql.Type { return sql.Int32 } // Eval implements the Expression interface. func (y *Year) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { - span, ctx := ctx.Span("function.Year") - defer span.Finish() return getDatePart(ctx, y.UnaryExpression, row, year) } @@ -83,8 +81,6 @@ func (m *Month) Type() sql.Type { return sql.Int32 } // Eval implements the Expression interface. func (m *Month) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { - span, ctx := ctx.Span("function.Month") - defer span.Finish() return getDatePart(ctx, m.UnaryExpression, row, month) } @@ -115,8 +111,6 @@ func (d *Day) Type() sql.Type { return sql.Int32 } // Eval implements the Expression interface. func (d *Day) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { - span, ctx := ctx.Span("function.Day") - defer span.Finish() return getDatePart(ctx, d.UnaryExpression, row, day) } @@ -147,8 +141,6 @@ func (h *Hour) Type() sql.Type { return sql.Int32 } // Eval implements the Expression interface. func (h *Hour) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { - span, ctx := ctx.Span("function.Hour") - defer span.Finish() return getDatePart(ctx, h.UnaryExpression, row, hour) } @@ -179,8 +171,6 @@ func (m *Minute) Type() sql.Type { return sql.Int32 } // Eval implements the Expression interface. func (m *Minute) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { - span, ctx := ctx.Span("function.Minute") - defer span.Finish() return getDatePart(ctx, m.UnaryExpression, row, minute) } @@ -211,8 +201,6 @@ func (s *Second) Type() sql.Type { return sql.Int32 } // Eval implements the Expression interface. func (s *Second) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { - span, ctx := ctx.Span("function.Second") - defer span.Finish() return getDatePart(ctx, s.UnaryExpression, row, second) } @@ -243,8 +231,6 @@ func (d *DayOfYear) Type() sql.Type { return sql.Int32 } // Eval implements the Expression interface. func (d *DayOfYear) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { - span, ctx := ctx.Span("function.DayOfYear") - defer span.Finish() return getDatePart(ctx, d.UnaryExpression, row, dayOfYear) } diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/isnull.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/isnull.go index 6b6cdecb7..cf0c0c73d 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/isnull.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/isnull.go @@ -24,9 +24,6 @@ func (e *IsNull) IsNullable() bool { // Eval implements the Expression interface. func (e *IsNull) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { - span, ctx := ctx.Span("expression.IsNull") - defer span.Finish() - v, err := e.Child.Eval(ctx, row) if err != nil { return nil, err diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/logic.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/logic.go index d5e032f0c..008de91fe 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/logic.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/logic.go @@ -43,9 +43,6 @@ func (*And) Type() sql.Type { // Eval implements the Expression interface. func (a *And) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { - span, ctx := ctx.Span("expression.And") - defer span.Finish() - lval, err := a.Left.Eval(ctx, row) if err != nil { return nil, err @@ -107,9 +104,6 @@ func (*Or) Type() sql.Type { // Eval implements the Expression interface. func (o *Or) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { - span, ctx := ctx.Span("expression.Or") - defer span.Finish() - lval, err := o.Left.Eval(ctx, row) if err != nil { return nil, err diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/tuple.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/tuple.go index 799103013..3f3dbaf19 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/tuple.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/tuple.go @@ -4,7 +4,6 @@ import ( "fmt" "strings" - opentracing "github.com/opentracing/opentracing-go" "gopkg.in/src-d/go-mysql-server.v0/sql" ) @@ -19,9 +18,6 @@ func NewTuple(exprs ...sql.Expression) Tuple { // Eval implements the Expression interface. func (t Tuple) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { - span, ctx := ctx.Span("expression.Tuple", opentracing.Tag{Key: "elems", Value: len(t)}) - defer span.Finish() - if len(t) == 1 { return t[0].Eval(ctx, row) } diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index.go index 9a80f025b..4336a35d5 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index.go @@ -10,6 +10,9 @@ import ( "gopkg.in/src-d/go-errors.v1" ) +// IndexBatchSize is the number of rows to save at a time when creating indexes. +var IndexBatchSize = uint64(10000) + // IndexKeyValueIter is an iterator of index key values, that is, a tuple of // the values that will be index keys. type IndexKeyValueIter interface { diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index/pilosa/driver.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index/pilosa/driver.go index df53707e3..19804874e 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index/pilosa/driver.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index/pilosa/driver.go @@ -7,7 +7,9 @@ import ( "os" "path/filepath" "strings" + "time" + opentracing "github.com/opentracing/opentracing-go" pilosa "github.com/pilosa/go-pilosa" "github.com/sirupsen/logrus" errors "gopkg.in/src-d/go-errors.v0" @@ -22,14 +24,18 @@ const ( IndexNamePrefix = "idx" // FrameNamePrefix the pilosa's frames prefix FrameNamePrefix = "frm" - // BatchSize is the number of objects to save when creating indexes. - BatchSize = 10000 ) // Driver implements sql.IndexDriver interface. type Driver struct { root string client *pilosa.Client + + // used for saving + bitBatches []*bitBatch + frames []*pilosa.Frame + timePilosa time.Duration + timeMapping time.Duration } // NewDriver returns a new instance of pilosa.Driver @@ -142,54 +148,14 @@ var ( errDeletePilosaIndex = errors.NewKind("error deleting pilosa index %s: %s") ) -type bitBatch struct { - size uint64 - bits []pilosa.Bit - pos uint64 -} - -func newBitBatch(size uint64) *bitBatch { - b := &bitBatch{size: size} - b.Clean() - - return b -} - -func (b *bitBatch) Clean() { - b.bits = make([]pilosa.Bit, 0, b.size) - b.pos = 0 -} - -func (b *bitBatch) Add(row, col uint64) { - b.bits = append(b.bits, pilosa.Bit{ - RowID: row, - ColumnID: col, - }) -} - -func (b *bitBatch) NextRecord() (pilosa.Record, error) { - if b.pos >= uint64(len(b.bits)) { - return nil, io.EOF - } - - b.pos++ - return b.bits[b.pos-1], nil -} - -func (b *bitBatch) Send(frame *pilosa.Frame, client *pilosa.Client) error { - return client.ImportFrame(frame, b) -} - // Save the given index (mapping and bitmap) func (d *Driver) Save( ctx *sql.Context, i sql.Index, iter sql.IndexKeyValueIter, ) (err error) { - span, ctx := ctx.Span("pilosa.Save") - span.LogKV("name", i.ID()) - - defer span.Finish() + var colID uint64 + start := time.Now() idx, ok := i.(*pilosaIndex) if !ok { @@ -223,9 +189,9 @@ func (d *Driver) Save( return errDeletePilosaIndex.New(pilosaIndex.Name(), err) } - frames := make([]*pilosa.Frame, len(idx.ExpressionHashes())) + d.frames = make([]*pilosa.Frame, len(idx.ExpressionHashes())) for i, e := range idx.ExpressionHashes() { - frames[i], err = pilosaIndex.Frame(frameName(e)) + d.frames[i], err = pilosaIndex.Frame(frameName(e)) if err != nil { return err } @@ -245,7 +211,7 @@ func (d *Driver) Save( if rollback { idx.mapping.rollback() } else { - e := idx.mapping.commit(false) + e := d.saveMapping(ctx, idx.mapping, colID, false) if e != nil && err == nil { err = e } @@ -254,27 +220,15 @@ func (d *Driver) Save( idx.mapping.close() }() - bitBatch := make([]*bitBatch, len(frames)) - for i := range bitBatch { - bitBatch[i] = newBitBatch(BatchSize) + d.bitBatches = make([]*bitBatch, len(d.frames)) + for i := range d.bitBatches { + d.bitBatches[i] = newBitBatch(sql.IndexBatchSize) } - for colID := uint64(0); err == nil; colID++ { + for colID = uint64(0); err == nil; colID++ { // commit each batch of objects (pilosa and boltdb) - if colID%BatchSize == 0 && colID != 0 { - for i, frm := range frames { - err = bitBatch[i].Send(frm, d.client) - if err != nil { - return err - } - - bitBatch[i].Clean() - } - - err = idx.mapping.commit(true) - if err != nil { - return err - } + if colID%sql.IndexBatchSize == 0 && colID != 0 { + d.saveBatch(ctx, idx.mapping, colID) } select { @@ -291,7 +245,7 @@ func (d *Driver) Save( break } - for i, frm := range frames { + for i, frm := range d.frames { if values[i] == nil { continue } @@ -301,7 +255,7 @@ func (d *Driver) Save( return err } - bitBatch[i].Add(rowID, colID) + d.bitBatches[i].Add(rowID, colID) } err = idx.mapping.putLocation(pilosaIndex.Name(), colID, location) } @@ -313,13 +267,19 @@ func (d *Driver) Save( rollback = false - for i, frm := range frames { - err = bitBatch[i].Send(frm, d.client) - if err != nil { - return err - } + err = d.savePilosa(ctx, colID) + if err != nil { + return err } + logrus.WithFields(logrus.Fields{ + "duration": time.Since(start), + "pilosa": d.timePilosa, + "mapping": d.timeMapping, + "rows": colID, + "id": i.ID(), + }).Debugf("finished pilosa indexing") + return index.RemoveProcessingFile(path) } @@ -338,6 +298,98 @@ func (d *Driver) Delete(idx sql.Index) error { return d.client.DeleteIndex(index) } +func (d *Driver) saveBatch(ctx *sql.Context, m *mapping, colID uint64) error { + err := d.savePilosa(ctx, colID) + if err != nil { + return err + } + + return d.saveMapping(ctx, m, colID, true) +} + +func (d *Driver) savePilosa(ctx *sql.Context, colID uint64) error { + span, _ := ctx.Span("pilosa.Save.bitBatch", + opentracing.Tag{Key: "cols", Value: colID}, + opentracing.Tag{Key: "frames", Value: len(d.frames)}, + ) + defer span.Finish() + + start := time.Now() + + for i, frm := range d.frames { + err := d.client.ImportFrame(frm, d.bitBatches[i]) + if err != nil { + span.LogKV("error", err) + return err + } + + d.bitBatches[i].Clean() + } + + d.timePilosa += time.Since(start) + + return nil +} + +func (d *Driver) saveMapping( + ctx *sql.Context, + m *mapping, + colID uint64, + cont bool, +) error { + span, _ := ctx.Span("pilosa.Save.mapping", + opentracing.Tag{Key: "cols", Value: colID}, + opentracing.Tag{Key: "continues", Value: cont}, + ) + defer span.Finish() + + start := time.Now() + + err := m.commit(cont) + if err != nil { + span.LogKV("error", err) + return err + } + + d.timeMapping += time.Since(start) + + return nil +} + +type bitBatch struct { + size uint64 + bits []pilosa.Bit + pos uint64 +} + +func newBitBatch(size uint64) *bitBatch { + b := &bitBatch{size: size} + b.Clean() + + return b +} + +func (b *bitBatch) Clean() { + b.bits = make([]pilosa.Bit, 0, b.size) + b.pos = 0 +} + +func (b *bitBatch) Add(row, col uint64) { + b.bits = append(b.bits, pilosa.Bit{ + RowID: row, + ColumnID: col, + }) +} + +func (b *bitBatch) NextRecord() (pilosa.Record, error) { + if b.pos >= uint64(len(b.bits)) { + return nil, io.EOF + } + + b.pos++ + return b.bits[b.pos-1], nil +} + func indexName(db, table, id string) string { h := sha1.New() io.WriteString(h, db) diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index/pilosa/driver_test.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index/pilosa/driver_test.go index 2e1f8d0e6..380c3bf2c 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index/pilosa/driver_test.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index/pilosa/driver_test.go @@ -143,7 +143,7 @@ func TestSaveAndLoad(t *testing.T) { found := false for _, span := range tracer.Spans { - if span == "pilosa.Save" { + if span == "pilosa.Save.bitBatch" { found = true break } diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index/pilosa/index.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index/pilosa/index.go index 6646496cc..1049653bb 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index/pilosa/index.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index/pilosa/index.go @@ -9,6 +9,7 @@ import ( "gopkg.in/src-d/go-errors.v1" + "github.com/boltdb/bolt" pilosa "github.com/pilosa/go-pilosa" "github.com/sirupsen/logrus" "gopkg.in/src-d/go-mysql-server.v0/sql" @@ -471,14 +472,33 @@ type indexValueIter struct { bits []uint64 mapping *mapping indexName string + + // share transaction and bucket on all getLocation calls + bucket *bolt.Bucket + tx *bolt.Tx } func (it *indexValueIter) Next() ([]byte, error) { + if it.bucket == nil { + bucket, err := it.mapping.getBucket(it.indexName, false) + if err != nil { + return nil, err + } + + it.bucket = bucket + it.tx = bucket.Tx() + } + if it.offset >= it.total { if err := it.Close(); err != nil { logrus.WithField("err", err.Error()). Error("unable to close the pilosa index value iterator") } + + if it.tx != nil { + it.tx.Rollback() + } + return nil, io.EOF } @@ -491,10 +511,16 @@ func (it *indexValueIter) Next() ([]byte, error) { it.offset++ - return it.mapping.getLocation(it.indexName, colID) + return it.mapping.getLocationFromBucket(it.bucket, colID) } -func (it *indexValueIter) Close() error { return it.mapping.close() } +func (it *indexValueIter) Close() error { + if it.tx != nil { + it.tx.Rollback() + } + + return it.mapping.close() +} var ( errUnknownType = errors.NewKind("unknown type %T received as value") diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index/pilosa/mapping.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index/pilosa/mapping.go index 9f3bb46c2..1d0581b52 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index/pilosa/mapping.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index/pilosa/mapping.go @@ -294,6 +294,47 @@ func (m *mapping) getLocation(indexName string, colID uint64) ([]byte, error) { return location, err } +func (m *mapping) getLocationFromBucket( + bucket *bolt.Bucket, + colID uint64, +) ([]byte, error) { + var location []byte + + err := m.query(func() error { + key := make([]byte, 8) + binary.LittleEndian.PutUint64(key, colID) + + location = bucket.Get(key) + return nil + }) + + return location, err +} + +func (m *mapping) getBucket( + indexName string, + writable bool, +) (*bolt.Bucket, error) { + var bucket *bolt.Bucket + + err := m.query(func() error { + tx, err := m.db.Begin(writable) + if err != nil { + return err + } + + bucket = tx.Bucket([]byte(indexName)) + if bucket == nil { + tx.Rollback() + return fmt.Errorf("bucket %s not found", indexName) + } + + return nil + }) + + return bucket, err +} + func (m *mapping) getLocationN(indexName string) (int, error) { var n int err := m.query(func() error { diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/plan/create_index.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/plan/create_index.go index 217e6d859..6ad7c52c4 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/plan/create_index.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/plan/create_index.go @@ -139,14 +139,16 @@ func (c *CreateIndex) backgroundIndexCreate( iter sql.IndexKeyValueIter, done chan<- struct{}, ) { - span, ctx := ctx.Span("plan.backgroundIndexCreate") - span.LogKV( - "index", index.ID(), - "table", index.Table(), - "driver", index.Driver(), - ) + span, ctx := ctx.Span("plan.backgroundIndexCreate", + opentracing.Tags{ + "index": index.ID(), + "table": index.Table(), + "driver": index.Driver(), + }) - err := driver.Save(ctx, index, newLoggingKeyValueIter(span, log, iter)) + l := log.WithField("id", index.ID()) + + err := driver.Save(ctx, index, newLoggingKeyValueIter(ctx, l, iter)) close(done) if err != nil { @@ -337,6 +339,7 @@ func getIndexKeyValueIter(ctx *sql.Context, table sql.Indexable, columns []strin } type loggingKeyValueIter struct { + ctx *sql.Context span opentracing.Span log *logrus.Entry iter sql.IndexKeyValueIter @@ -345,12 +348,12 @@ type loggingKeyValueIter struct { } func newLoggingKeyValueIter( - span opentracing.Span, + ctx *sql.Context, log *logrus.Entry, iter sql.IndexKeyValueIter, ) sql.IndexKeyValueIter { return &loggingKeyValueIter{ - span: span, + ctx: ctx, log: log, iter: iter, start: time.Now(), @@ -358,23 +361,40 @@ func newLoggingKeyValueIter( } func (i *loggingKeyValueIter) Next() ([]interface{}, []byte, error) { + if i.span == nil { + i.span, _ = i.ctx.Span("plan.backgroundIndexCreate.iterator", + opentracing.Tags{ + "start": i.rows, + }, + ) + } + i.rows++ - if i.rows%100 == 0 { + if i.rows%sql.IndexBatchSize == 0 { duration := time.Since(i.start) - i.log.WithField("duration", duration). - Debugf("still creating index: %d rows saved so far", i.rows) + i.log.WithFields(logrus.Fields{ + "duration": duration, + "rows": i.rows, + }).Debugf("still creating index") - i.span.LogFields( - otlog.String("event", "saved rows"), - otlog.Uint64("rows", i.rows), - otlog.String("duration", duration.String()), - ) + if i.span != nil { + i.span.LogKV("duration", duration.String()) + i.span.Finish() + i.span = nil + } i.start = time.Now() } - return i.iter.Next() + val, loc, err := i.iter.Next() + if err != nil { + i.span.LogKV("error", err) + i.span.Finish() + i.span = nil + } + + return val, loc, err } func (i *loggingKeyValueIter) Close() error {