From 4dfa8acef09cfa33b94bfdc6193629e9700fb3e2 Mon Sep 17 00:00:00 2001 From: Artem Date: Thu, 23 Nov 2023 18:45:35 +0100 Subject: [PATCH] Feature: gas tracker endpoint --- cmd/api/docs/docs.go | 91 ++++++++++++++++ cmd/api/docs/swagger.json | 91 ++++++++++++++++ cmd/api/docs/swagger.yaml | 66 +++++++++++ cmd/api/handler/gas.go | 181 ++++++++++++++++++++++++++++++- cmd/api/handler/gas_test.go | 16 ++- cmd/api/handler/responses/gas.go | 24 ++++ cmd/api/init.go | 3 +- internal/storage/mock/tx.go | 40 +++++++ internal/storage/postgres/tx.go | 10 ++ internal/storage/tx.go | 14 +++ 10 files changed, 529 insertions(+), 7 deletions(-) create mode 100644 cmd/api/handler/responses/gas.go diff --git a/cmd/api/docs/docs.go b/cmd/api/docs/docs.go index e7f4700b..b2b71981 100644 --- a/cmd/api/docs/docs.go +++ b/cmd/api/docs/docs.go @@ -1030,6 +1030,36 @@ const docTemplate = `{ } } }, + "/v1/gas/price": { + "get": { + "description": "Get estimated gas price based on historical data", + "produces": [ + "application/json" + ], + "tags": [ + "gas" + ], + "summary": "Get estimated gas price", + "operationId": "gas-price", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/responses.GasPrice" + } + }, + "204": { + "description": "No Content" + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handler.Error" + } + } + } + } + }, "/v1/head": { "get": { "description": "Get current indexer head", @@ -2836,6 +2866,67 @@ const docTemplate = `{ } } }, + "responses.GasBlock": { + "type": "object", + "properties": { + "avg_gas_price": { + "type": "string", + "format": "string", + "example": "0.12345" + }, + "gas_used_ratio": { + "type": "string", + "format": "string", + "example": "0.12345" + }, + "height": { + "type": "integer", + "format": "int64", + "example": 12345 + }, + "total_fee": { + "type": "string", + "format": "string", + "example": "1972367126" + }, + "total_gas_used": { + "type": "integer", + "format": "int64", + "example": 56789 + }, + "total_gas_wanted": { + "type": "integer", + "format": "int64", + "example": 86756 + } + } + }, + "responses.GasPrice": { + "type": "object", + "properties": { + "computed_blocks": { + "type": "array", + "items": { + "$ref": "#/definitions/responses.GasBlock" + } + }, + "fast": { + "type": "string", + "format": "string", + "example": "0.1234" + }, + "median": { + "type": "string", + "format": "string", + "example": "0.1234" + }, + "slow": { + "type": "string", + "format": "string", + "example": "0.1234" + } + } + }, "responses.HistogramItem": { "type": "object", "properties": { diff --git a/cmd/api/docs/swagger.json b/cmd/api/docs/swagger.json index 1ffc4138..c1adea43 100644 --- a/cmd/api/docs/swagger.json +++ b/cmd/api/docs/swagger.json @@ -1020,6 +1020,36 @@ } } }, + "/v1/gas/price": { + "get": { + "description": "Get estimated gas price based on historical data", + "produces": [ + "application/json" + ], + "tags": [ + "gas" + ], + "summary": "Get estimated gas price", + "operationId": "gas-price", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/responses.GasPrice" + } + }, + "204": { + "description": "No Content" + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handler.Error" + } + } + } + } + }, "/v1/head": { "get": { "description": "Get current indexer head", @@ -2826,6 +2856,67 @@ } } }, + "responses.GasBlock": { + "type": "object", + "properties": { + "avg_gas_price": { + "type": "string", + "format": "string", + "example": "0.12345" + }, + "gas_used_ratio": { + "type": "string", + "format": "string", + "example": "0.12345" + }, + "height": { + "type": "integer", + "format": "int64", + "example": 12345 + }, + "total_fee": { + "type": "string", + "format": "string", + "example": "1972367126" + }, + "total_gas_used": { + "type": "integer", + "format": "int64", + "example": 56789 + }, + "total_gas_wanted": { + "type": "integer", + "format": "int64", + "example": 86756 + } + } + }, + "responses.GasPrice": { + "type": "object", + "properties": { + "computed_blocks": { + "type": "array", + "items": { + "$ref": "#/definitions/responses.GasBlock" + } + }, + "fast": { + "type": "string", + "format": "string", + "example": "0.1234" + }, + "median": { + "type": "string", + "format": "string", + "example": "0.1234" + }, + "slow": { + "type": "string", + "format": "string", + "example": "0.1234" + } + } + }, "responses.HistogramItem": { "type": "object", "properties": { diff --git a/cmd/api/docs/swagger.yaml b/cmd/api/docs/swagger.yaml index ab49b98e..50c7edd7 100644 --- a/cmd/api/docs/swagger.yaml +++ b/cmd/api/docs/swagger.yaml @@ -232,6 +232,52 @@ definitions: - $ref: '#/definitions/types.EventType' example: commission type: object + responses.GasBlock: + properties: + avg_gas_price: + example: "0.12345" + format: string + type: string + gas_used_ratio: + example: "0.12345" + format: string + type: string + height: + example: 12345 + format: int64 + type: integer + total_fee: + example: "1972367126" + format: string + type: string + total_gas_used: + example: 56789 + format: int64 + type: integer + total_gas_wanted: + example: 86756 + format: int64 + type: integer + type: object + responses.GasPrice: + properties: + computed_blocks: + items: + $ref: '#/definitions/responses.GasBlock' + type: array + fast: + example: "0.1234" + format: string + type: string + median: + example: "0.1234" + format: string + type: string + slow: + example: "0.1234" + format: string + type: string + type: object responses.HistogramItem: properties: time: @@ -1634,6 +1680,26 @@ paths: summary: Get estimated gas for pay for blob tags: - gas + /v1/gas/price: + get: + description: Get estimated gas price based on historical data + operationId: gas-price + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/responses.GasPrice' + "204": + description: No Content + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/handler.Error' + summary: Get estimated gas price + tags: + - gas /v1/head: get: description: Get current indexer head diff --git a/cmd/api/handler/gas.go b/cmd/api/handler/gas.go index fe185d59..7c05c5c1 100644 --- a/cmd/api/handler/gas.go +++ b/cmd/api/handler/gas.go @@ -4,18 +4,39 @@ package handler import ( + "context" "net/http" + "sort" "strconv" + "sync" + "github.com/celenium-io/celestia-indexer/cmd/api/handler/responses" + "github.com/celenium-io/celestia-indexer/internal/storage" + "github.com/celenium-io/celestia-indexer/pkg/types" + "github.com/celestiaorg/celestia-app/pkg/appconsts" blobtypes "github.com/celestiaorg/celestia-app/x/blob/types" + sdk "github.com/dipdup-net/indexer-sdk/pkg/storage" "github.com/labstack/echo/v4" + "github.com/shopspring/decimal" ) // GasHandler - -type GasHandler struct{} +type GasHandler struct { + state storage.IState + tx storage.ITx + blockStats storage.IBlockStats +} -func NewGasHandler() GasHandler { - return GasHandler{} +func NewGasHandler( + state storage.IState, + tx storage.ITx, + blockStats storage.IBlockStats, +) GasHandler { + return GasHandler{ + state: state, + tx: tx, + blockStats: blockStats, + } } type estimatePfbGas struct { @@ -49,3 +70,157 @@ func (handler GasHandler) EstimateForPfb(c echo.Context) error { return c.JSON(http.StatusOK, blobtypes.DefaultEstimateGas(sizes)) } + +const ( + estimationGasPriceBlocksCount = 5 +) + +// EstimatePrice godoc +// +// @Summary Get estimated gas price +// @Description Get estimated gas price based on historical data +// @Tags gas +// @ID gas-price +// @Produce json +// @Success 200 {object} responses.GasPrice +// @Success 204 +// @Failure 500 {object} Error +// @Router /v1/gas/price [get] +func (handler GasHandler) EstimatePrice(c echo.Context) error { + ctx := c.Request().Context() + states, err := handler.state.List(ctx, 1, 0, sdk.SortOrderAsc) + if err != nil { + return handleError(c, err, handler.state) + } + if len(states) == 0 { + return c.JSON(http.StatusNoContent, []any{}) + } + state := states[0] + lastBlockHeight := state.LastHeight - estimationGasPriceBlocksCount + + var ( + wg sync.WaitGroup + result = make(chan gasPrice, estimationGasPriceBlocksCount) + errs = make(chan error) + ) + for height := state.LastHeight; height > lastBlockHeight; height-- { + wg.Add(1) + go handler.computeGasPriceEstimationForBlock(ctx, height, result, errs, &wg) + } + wg.Wait() + + gas := newGasPrice() + + for { + select { + case <-ctx.Done(): + return c.JSON(http.StatusOK, Error{ + Message: ctx.Err().Error(), + }) + case err := <-errs: + return internalServerError(c, err) + case gp := <-result: + gas.percentiles[0] = gas.percentiles[0].Add(gp.percentiles[0]) + gas.percentiles[1] = gas.percentiles[1].Add(gp.percentiles[1]) + gas.percentiles[2] = gas.percentiles[2].Add(gp.percentiles[2]) + + gas.blocks = append(gas.blocks, responses.GasBlock{ + Height: uint64(gp.stats.Height), + GasWanted: uint64(gp.stats.GasLimit), + GasUsed: uint64(gp.stats.GasUsed), + Fee: gp.stats.Fee.String(), + GasPrice: gp.stats.Fee.Div(decimal.NewFromInt(gp.stats.GasLimit)).String(), + GasUsedRatio: decimal.NewFromInt(gp.stats.GasUsed).Div(decimal.NewFromInt(gp.stats.GasLimit)).String(), + TxCount: uint64(gp.stats.TxCount), + Percentiles: []string{ + gp.percentiles[0].String(), + gp.percentiles[1].String(), + gp.percentiles[2].String(), + }, + }) + + if len(result) == 0 { + gas.percentiles[0] = gas.percentiles[0].Div(decimal.NewFromInt(estimationGasPriceBlocksCount)) + gas.percentiles[1] = gas.percentiles[1].Div(decimal.NewFromInt(estimationGasPriceBlocksCount)) + gas.percentiles[2] = gas.percentiles[2].Div(decimal.NewFromInt(estimationGasPriceBlocksCount)) + + return c.JSON(http.StatusOK, gas.toResponse()) + } + } + } +} + +type gasPrice struct { + percentiles []decimal.Decimal + stats storage.BlockStats + + blocks []responses.GasBlock +} + +func newGasPrice() gasPrice { + return gasPrice{ + percentiles: []decimal.Decimal{ + decimal.New(0, 1), + decimal.New(0, 1), + decimal.New(0, 1), + }, + blocks: make([]responses.GasBlock, 0), + } +} + +func (gp gasPrice) toResponse() responses.GasPrice { + return responses.GasPrice{ + Slow: gp.percentiles[0].String(), + Median: gp.percentiles[1].String(), + Fast: gp.percentiles[2].String(), + ComputedBlocks: gp.blocks, + } +} + +var ( + minGasPrice = decimal.NewFromFloat(appconsts.DefaultMinGasPrice) +) + +func (handler GasHandler) computeGasPriceEstimationForBlock(ctx context.Context, height types.Level, result chan<- gasPrice, errs chan<- error, wg *sync.WaitGroup) { + defer wg.Done() + + block, err := handler.blockStats.ByHeight(ctx, height) + if err != nil { + errs <- err + return + } + if block.TxCount == 0 { + result <- newGasPrice() + return + } + + txs, err := handler.tx.Gas(ctx, height) + if err != nil { + errs <- err + return + } + sort.Sort(storage.ByGasPrice(txs)) + + var ( + gp = newGasPrice() + sumGas = txs[0].GasWanted + txIndex = 0 + ) + + gp.stats = block + + for i, p := range []float64{.10, .50, .99} { + threshold := uint64(float64(block.GasLimit) * p) + for sumGas < int64(threshold) && txIndex < len(txs) { + txIndex++ + sumGas += txs[txIndex].GasWanted + } + if txs[txIndex].GasPrice.LessThan(minGasPrice) { + gp.percentiles[txIndex] = minGasPrice.Copy() + } else { + gp.percentiles[i] = txs[txIndex].GasPrice + } + } + + result <- gp +} diff --git a/cmd/api/handler/gas_test.go b/cmd/api/handler/gas_test.go index 2beb1bb3..443449fb 100644 --- a/cmd/api/handler/gas_test.go +++ b/cmd/api/handler/gas_test.go @@ -11,22 +11,32 @@ import ( "net/url" "testing" + "github.com/celenium-io/celestia-indexer/internal/storage/mock" "github.com/labstack/echo/v4" "github.com/stretchr/testify/suite" + "go.uber.org/mock/gomock" ) // GasTestSuite - type GasTestSuite struct { suite.Suite - echo *echo.Echo - handler GasHandler + echo *echo.Echo + state *mock.MockIState + txs *mock.MockITx + blockStats *mock.MockIBlockStats + handler GasHandler + ctrl *gomock.Controller } // SetupSuite - func (s *GasTestSuite) SetupSuite() { s.echo = echo.New() s.echo.Validator = NewCelestiaApiValidator() - s.handler = NewGasHandler() + s.ctrl = gomock.NewController(s.T()) + s.state = mock.NewMockIState(s.ctrl) + s.txs = mock.NewMockITx(s.ctrl) + s.blockStats = mock.NewMockIBlockStats(s.ctrl) + s.handler = NewGasHandler(s.state, s.txs, s.blockStats) } // TearDownSuite - diff --git a/cmd/api/handler/responses/gas.go b/cmd/api/handler/responses/gas.go new file mode 100644 index 00000000..56615c69 --- /dev/null +++ b/cmd/api/handler/responses/gas.go @@ -0,0 +1,24 @@ +// SPDX-FileCopyrightText: 2023 PK Lab AG +// SPDX-License-Identifier: MIT + +package responses + +type GasPrice struct { + Slow string `example:"0.1234" format:"string" json:"slow" swaggertype:"string"` + Median string `example:"0.1234" format:"string" json:"median" swaggertype:"string"` + Fast string `example:"0.1234" format:"string" json:"fast" swaggertype:"string"` + + ComputedBlocks []GasBlock `json:"computed_blocks"` +} + +type GasBlock struct { + Height uint64 `example:"12345" format:"int64" json:"height" swaggertype:"integer"` + GasWanted uint64 `example:"86756" format:"int64" json:"total_gas_wanted" swaggertype:"integer"` + GasUsed uint64 `example:"56789" format:"int64" json:"total_gas_used" swaggertype:"integer"` + TxCount uint64 `example:"12" format:"int64" json:"tx_count" swaggertype:"integer"` + Fee string `example:"1972367126" format:"string" json:"total_fee" swaggertype:"string"` + GasPrice string `example:"0.12345" format:"string" json:"avg_gas_price" swaggertype:"string"` + GasUsedRatio string `example:"0.12345" format:"string" json:"gas_used_ratio" swaggertype:"string"` + + Percentiles []string `json:"percentiles"` +} diff --git a/cmd/api/init.go b/cmd/api/init.go index a6c9d97e..47338c5d 100644 --- a/cmd/api/init.go +++ b/cmd/api/init.go @@ -331,10 +331,11 @@ func initHandlers(ctx context.Context, e *echo.Echo, cfg Config, db postgres.Sto } } - gasHandler := handler.NewGasHandler() + gasHandler := handler.NewGasHandler(db.State, db.Tx, db.BlockStats) gas := v1.Group("/gas") { gas.GET("/estimate_for_pfb", gasHandler.EstimateForPfb) + gas.GET("/price", gasHandler.EstimatePrice) } if cfg.ApiConfig.Prometheus { diff --git a/internal/storage/mock/tx.go b/internal/storage/mock/tx.go index 8b9c298a..7429694d 100644 --- a/internal/storage/mock/tx.go +++ b/internal/storage/mock/tx.go @@ -16,6 +16,7 @@ import ( reflect "reflect" storage "github.com/celenium-io/celestia-indexer/internal/storage" + types "github.com/celenium-io/celestia-indexer/pkg/types" storage0 "github.com/dipdup-net/indexer-sdk/pkg/storage" gomock "go.uber.org/mock/gomock" ) @@ -238,6 +239,45 @@ func (c *ITxFilterCall) DoAndReturn(f func(context.Context, storage.TxFilter) ([ return c } +// Gas mocks base method. +func (m *MockITx) Gas(ctx context.Context, height types.Level) ([]storage.Gas, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Gas", ctx, height) + ret0, _ := ret[0].([]storage.Gas) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Gas indicates an expected call of Gas. +func (mr *MockITxMockRecorder) Gas(ctx, height any) *ITxGasCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Gas", reflect.TypeOf((*MockITx)(nil).Gas), ctx, height) + return &ITxGasCall{Call: call} +} + +// ITxGasCall wrap *gomock.Call +type ITxGasCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *ITxGasCall) Return(arg0 []storage.Gas, arg1 error) *ITxGasCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *ITxGasCall) Do(f func(context.Context, types.Level) ([]storage.Gas, error)) *ITxGasCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *ITxGasCall) DoAndReturn(f func(context.Context, types.Level) ([]storage.Gas, error)) *ITxGasCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + // Genesis mocks base method. func (m *MockITx) Genesis(ctx context.Context, limit, offset int, sortOrder storage0.SortOrder) ([]storage.Tx, error) { m.ctrl.T.Helper() diff --git a/internal/storage/postgres/tx.go b/internal/storage/postgres/tx.go index 46a99f8b..11bebef1 100644 --- a/internal/storage/postgres/tx.go +++ b/internal/storage/postgres/tx.go @@ -7,6 +7,7 @@ import ( "context" "github.com/celenium-io/celestia-indexer/internal/storage" + "github.com/celenium-io/celestia-indexer/pkg/types" "github.com/dipdup-net/go-lib/database" sdk "github.com/dipdup-net/indexer-sdk/pkg/storage" "github.com/dipdup-net/indexer-sdk/pkg/storage/postgres" @@ -75,3 +76,12 @@ func (tx *Tx) Genesis(ctx context.Context, limit, offset int, sortOrder sdk.Sort err = query.Scan(ctx) return } + +func (tx *Tx) Gas(ctx context.Context, height types.Level) (response []storage.Gas, err error) { + err = tx.DB().NewSelect(). + Model((*storage.Tx)(nil)). + ColumnExpr("gas_wanted, gas_used, fee, (CASE WHEN gas_wanted > 0 THEN fee / gas_wanted ELSE 0 END) as gas_price"). + Where("height = ?", height). + Scan(ctx, &response) + return +} diff --git a/internal/storage/tx.go b/internal/storage/tx.go index af46dd7a..34dad98e 100644 --- a/internal/storage/tx.go +++ b/internal/storage/tx.go @@ -24,8 +24,22 @@ type ITx interface { ByIdWithRelations(ctx context.Context, id uint64) (Tx, error) ByAddress(ctx context.Context, addressId uint64, fltrs TxFilter) ([]Tx, error) Genesis(ctx context.Context, limit, offset int, sortOrder storage.SortOrder) ([]Tx, error) + Gas(ctx context.Context, height pkgTypes.Level) ([]Gas, error) } +type Gas struct { + GasWanted int64 `bun:"gas_wanted"` + GasUsed int64 `bun:"gas_used"` + Fee decimal.Decimal `bun:"fee"` + GasPrice decimal.Decimal `bun:"gas_price"` +} + +type ByGasPrice []Gas + +func (gp ByGasPrice) Len() int { return len(gp) } +func (gp ByGasPrice) Less(i, j int) bool { return gp[j].GasPrice.GreaterThan(gp[i].GasPrice) } +func (gp ByGasPrice) Swap(i, j int) { gp[i], gp[j] = gp[j], gp[i] } + type TxFilter struct { Limit int Offset int