From ce7f2a20ad72fc22d48e076454498af312e1974b Mon Sep 17 00:00:00 2001 From: Artem Date: Mon, 20 Nov 2023 18:59:46 +0100 Subject: [PATCH] Feature: block stats views --- cmd/api/docs/docs.go | 244 ++++++++++++------ cmd/api/docs/swagger.json | 244 ++++++++++++------ cmd/api/docs/swagger.yaml | 181 ++++++++----- cmd/api/handler/responses/block.go | 4 + cmd/api/handler/responses/stats.go | 45 ++-- cmd/api/handler/responses/stats_test.go | 39 --- cmd/api/handler/stats.go | 66 +++-- cmd/api/handler/stats_test.go | 95 ++++--- cmd/api/init.go | 8 +- cmd/api/main.go | 10 +- cmd/indexer/main.go | 2 +- database/views/00_block_stats_by_minute.sql | 35 +++ database/views/01_block_stats_by_hour.sql | 35 +++ database/views/02_block_stats_by_day.sql | 35 +++ database/views/03_block_stats_by_year.sql | 35 +++ database/views/04_block_stats_by_month.sql | 35 +++ database/views/05_block_stats_by_week.sql | 35 +++ .../views/gas_price_candlesticks_hourly.sql | 10 - database/views/tx_count_hourly.sql | 8 - internal/storage/block_stats.go | 2 + internal/storage/mock/stats.go | 76 +++--- internal/storage/postgres/stats.go | 75 +++++- internal/storage/postgres/stats_test.go | 4 +- internal/storage/postgres/storage_test.go | 4 +- internal/storage/postgres/views.go | 17 +- internal/storage/stats.go | 36 ++- internal/storage/views.go | 8 +- pkg/indexer/parser/parse.go | 2 + test/data/block_stats.yml | 6 + test/data/rollback/block_stats.yml | 6 + 30 files changed, 964 insertions(+), 438 deletions(-) delete mode 100644 cmd/api/handler/responses/stats_test.go create mode 100644 database/views/00_block_stats_by_minute.sql create mode 100644 database/views/01_block_stats_by_hour.sql create mode 100644 database/views/02_block_stats_by_day.sql create mode 100644 database/views/03_block_stats_by_year.sql create mode 100644 database/views/04_block_stats_by_month.sql create mode 100644 database/views/05_block_stats_by_week.sql delete mode 100644 database/views/gas_price_candlesticks_hourly.sql delete mode 100644 database/views/tx_count_hourly.sql diff --git a/cmd/api/docs/docs.go b/cmd/api/docs/docs.go index 7818f00d..c905def3 100644 --- a/cmd/api/docs/docs.go +++ b/cmd/api/docs/docs.go @@ -1553,36 +1553,6 @@ const docTemplate = `{ } } }, - "/v1/stats/gas_price/hourly": { - "get": { - "description": "Get candles for gas price with volume, average gas efficiency and fee for an hour", - "produces": [ - "application/json" - ], - "tags": [ - "stats" - ], - "summary": "Get candles for gas price", - "operationId": "stats-gas-price-hourly", - "responses": { - "200": { - "description": "OK", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/responses.GasPriceCandle" - } - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/handler.Error" - } - } - } - } - }, "/v1/stats/histogram/{table}/{function}/{timeframe}": { "get": { "description": "Returns histogram by table, function and timeframe\n\n### Parameters\n\n` + "`" + `table` + "`" + `, ` + "`" + `function` + "`" + ` and ` + "`" + `column` + "`" + ` parameters are the same as summary endpoint.\n\n\n### Timeframe\n\n* ` + "`" + `hour` + "`" + `\n* ` + "`" + `day` + "`" + `\n* ` + "`" + `week` + "`" + `\n* ` + "`" + `month` + "`" + `\n* ` + "`" + `year` + "`" + `", @@ -1681,6 +1651,121 @@ const docTemplate = `{ } } }, + "/v1/stats/namespace/usage": { + "get": { + "description": "Get namespaces with sorting by size. Returns top 100 namespaces. Namespaces which is not included to top 100 grouped into 'others' item", + "produces": [ + "application/json" + ], + "tags": [ + "stats" + ], + "summary": "Get namespaces with sorting by size.", + "operationId": "stats-namespace-usage", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/responses.NamespaceUsage" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handler.Error" + } + } + } + } + }, + "/v1/stats/series/{name}/{timeframe}": { + "get": { + "description": "Get histogram with precomputed stats by series name and timeframe", + "produces": [ + "application/json" + ], + "tags": [ + "stats" + ], + "summary": "Get histogram with precomputed stats", + "operationId": "stats-series", + "parameters": [ + { + "enum": [ + "hour", + "day", + "week", + "month", + "year" + ], + "type": "string", + "description": "Timeframe", + "name": "timeframe", + "in": "path", + "required": true + }, + { + "enum": [ + "blobs_size", + "tps", + "bps", + "fee", + "supply_change", + "block_time", + "tx_count", + "events_count", + "gas_price", + "gas_efficiency", + "gas_used", + "gas_limit" + ], + "type": "string", + "description": "Series name", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "integer", + "description": "Time from in unix timestamp", + "name": "from", + "in": "query" + }, + { + "type": "integer", + "description": "Time to in unix timestamp", + "name": "to", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/responses.SeriesItem" + } + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/handler.Error" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handler.Error" + } + } + } + } + }, "/v1/stats/summary/{table}/{function}": { "get": { "description": "Returns string value by passed table and function.\n\n### Availiable tables\n* ` + "`" + `block` + "`" + `\n* ` + "`" + `block_stats` + "`" + `\n* ` + "`" + `tx` + "`" + `\n* ` + "`" + `message` + "`" + `\n* ` + "`" + `event` + "`" + `\n\n\n### Availiable functions\n* ` + "`" + `sum` + "`" + `\n* ` + "`" + `min` + "`" + `\n* ` + "`" + `max` + "`" + `\n* ` + "`" + `avg` + "`" + `\n* ` + "`" + `count` + "`" + `\n\n\n` + "`" + `Column` + "`" + ` query parameter is required for functions ` + "`" + `sum` + "`" + `, ` + "`" + `min` + "`" + `, ` + "`" + `max` + "`" + ` and ` + "`" + `avg` + "`" + ` and should not pass for ` + "`" + `count` + "`" + `.\n\n\n### Availiable columns and functions for tables:\n\n#### Block\n* ` + "`" + `height` + "`" + ` -- min max\n* ` + "`" + `time` + "`" + ` -- min max\n\n#### Block stats\n* ` + "`" + `height` + "`" + ` -- min max\n* ` + "`" + `time` + "`" + ` -- min max\n* ` + "`" + `tx_count` + "`" + ` -- min max sum avg\n* ` + "`" + `events_count` + "`" + ` -- min max sum avg\n* ` + "`" + `blobs_size` + "`" + ` -- min max sum avg\n* ` + "`" + `block_time` + "`" + ` -- min max sum avg\n* ` + "`" + `supply_chnge` + "`" + ` -- min max sum avg\n* ` + "`" + `inflation_rate` + "`" + ` -- min max avg\n* ` + "`" + `fee` + "`" + ` -- min max sum avg\n\n#### Tx\n* ` + "`" + `height` + "`" + ` -- min max\n* ` + "`" + `time` + "`" + ` -- min max\n* ` + "`" + `gas_wanted` + "`" + ` -- min max sum avg\n* ` + "`" + `gas_used` + "`" + ` -- min max sum avg\n* ` + "`" + `timeout_height` + "`" + ` -- min max avg\n* ` + "`" + `events_count` + "`" + ` -- min max sum avg\n* ` + "`" + `messages_count` + "`" + ` -- min max sum avg\n* ` + "`" + `fee` + "`" + ` -- min max sum avg\n\n#### Event\n* ` + "`" + `height` + "`" + ` -- min max\n* ` + "`" + `time` + "`" + ` -- min max\n\n#### Message\n* ` + "`" + `height` + "`" + ` -- min max\n* ` + "`" + `time` + "`" + ` -- min max", @@ -2522,6 +2607,14 @@ const docTemplate = `{ "type": "string", "example": "28347628346" }, + "gas_limit": { + "type": "integer", + "example": 1234 + }, + "gas_used": { + "type": "integer", + "example": 1234 + }, "inflation_rate": { "type": "string", "example": "0.0800000" @@ -2634,51 +2727,6 @@ const docTemplate = `{ } } }, - "responses.GasPriceCandle": { - "type": "object", - "properties": { - "avg_gas_efficiency": { - "type": "string", - "format": "string", - "example": "0.45282" - }, - "avg_gas_price": { - "type": "string", - "format": "string", - "example": "0.45282" - }, - "fee": { - "type": "number", - "format": "integer", - "example": 1283518 - }, - "high": { - "type": "string", - "format": "string", - "example": "0.17632" - }, - "low": { - "type": "string", - "format": "string", - "example": "0.11882" - }, - "time": { - "type": "string", - "format": "date-time", - "example": "2023-07-04T03:10:57+00:00" - }, - "total_gas_limit": { - "type": "string", - "format": "string", - "example": "1213134" - }, - "total_gas_used": { - "type": "string", - "format": "string", - "example": "0.45282" - } - } - }, "responses.HistogramItem": { "type": "object", "properties": { @@ -2842,12 +2890,52 @@ const docTemplate = `{ } } }, + "responses.NamespaceUsage": { + "type": "object", + "properties": { + "name": { + "type": "string", + "format": "string", + "example": "00112233" + }, + "size": { + "type": "number", + "format": "integer", + "example": 1283518 + } + } + }, "responses.Params": { "type": "object", "additionalProperties": { "type": "string" } }, + "responses.SeriesItem": { + "type": "object", + "properties": { + "max": { + "type": "string", + "format": "string", + "example": "0.17632" + }, + "min": { + "type": "string", + "format": "string", + "example": "0.17632" + }, + "time": { + "type": "string", + "format": "date-time", + "example": "2023-07-04T03:10:57+00:00" + }, + "value": { + "type": "string", + "format": "string", + "example": "0.17632" + } + } + }, "responses.State": { "type": "object", "properties": { @@ -3326,11 +3414,11 @@ const docTemplate = `{ // SwaggerInfo holds exported Swagger Info so clients can modify it var SwaggerInfo = &swag.Spec{ Version: "1.0", - Host: "api.celestia.dipdup.net", + Host: "api.celenium.io", BasePath: "", Schemes: []string{}, - Title: "Swagger Celestia Indexer API", - Description: "This is docs of Celestia indexer API.", + Title: "Swagger Celenium API", + Description: "This is docs of Celenium API.", InfoInstanceName: "swagger", SwaggerTemplate: docTemplate, LeftDelim: "{{", diff --git a/cmd/api/docs/swagger.json b/cmd/api/docs/swagger.json index ab745ce5..acf00695 100644 --- a/cmd/api/docs/swagger.json +++ b/cmd/api/docs/swagger.json @@ -1,12 +1,12 @@ { "swagger": "2.0", "info": { - "description": "This is docs of Celestia indexer API.", - "title": "Swagger Celestia Indexer API", + "description": "This is docs of Celenium API.", + "title": "Swagger Celenium API", "contact": {}, "version": "1.0" }, - "host": "api.celestia.dipdup.net", + "host": "api.celenium.io", "paths": { "/v1/address": { "get": { @@ -1546,36 +1546,6 @@ } } }, - "/v1/stats/gas_price/hourly": { - "get": { - "description": "Get candles for gas price with volume, average gas efficiency and fee for an hour", - "produces": [ - "application/json" - ], - "tags": [ - "stats" - ], - "summary": "Get candles for gas price", - "operationId": "stats-gas-price-hourly", - "responses": { - "200": { - "description": "OK", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/responses.GasPriceCandle" - } - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/handler.Error" - } - } - } - } - }, "/v1/stats/histogram/{table}/{function}/{timeframe}": { "get": { "description": "Returns histogram by table, function and timeframe\n\n### Parameters\n\n`table`, `function` and `column` parameters are the same as summary endpoint.\n\n\n### Timeframe\n\n* `hour`\n* `day`\n* `week`\n* `month`\n* `year`", @@ -1674,6 +1644,121 @@ } } }, + "/v1/stats/namespace/usage": { + "get": { + "description": "Get namespaces with sorting by size. Returns top 100 namespaces. Namespaces which is not included to top 100 grouped into 'others' item", + "produces": [ + "application/json" + ], + "tags": [ + "stats" + ], + "summary": "Get namespaces with sorting by size.", + "operationId": "stats-namespace-usage", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/responses.NamespaceUsage" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handler.Error" + } + } + } + } + }, + "/v1/stats/series/{name}/{timeframe}": { + "get": { + "description": "Get histogram with precomputed stats by series name and timeframe", + "produces": [ + "application/json" + ], + "tags": [ + "stats" + ], + "summary": "Get histogram with precomputed stats", + "operationId": "stats-series", + "parameters": [ + { + "enum": [ + "hour", + "day", + "week", + "month", + "year" + ], + "type": "string", + "description": "Timeframe", + "name": "timeframe", + "in": "path", + "required": true + }, + { + "enum": [ + "blobs_size", + "tps", + "bps", + "fee", + "supply_change", + "block_time", + "tx_count", + "events_count", + "gas_price", + "gas_efficiency", + "gas_used", + "gas_limit" + ], + "type": "string", + "description": "Series name", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "integer", + "description": "Time from in unix timestamp", + "name": "from", + "in": "query" + }, + { + "type": "integer", + "description": "Time to in unix timestamp", + "name": "to", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/responses.SeriesItem" + } + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/handler.Error" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handler.Error" + } + } + } + } + }, "/v1/stats/summary/{table}/{function}": { "get": { "description": "Returns string value by passed table and function.\n\n### Availiable tables\n* `block`\n* `block_stats`\n* `tx`\n* `message`\n* `event`\n\n\n### Availiable functions\n* `sum`\n* `min`\n* `max`\n* `avg`\n* `count`\n\n\n`Column` query parameter is required for functions `sum`, `min`, `max` and `avg` and should not pass for `count`.\n\n\n### Availiable columns and functions for tables:\n\n#### Block\n* `height` -- min max\n* `time` -- min max\n\n#### Block stats\n* `height` -- min max\n* `time` -- min max\n* `tx_count` -- min max sum avg\n* `events_count` -- min max sum avg\n* `blobs_size` -- min max sum avg\n* `block_time` -- min max sum avg\n* `supply_chnge` -- min max sum avg\n* `inflation_rate` -- min max avg\n* `fee` -- min max sum avg\n\n#### Tx\n* `height` -- min max\n* `time` -- min max\n* `gas_wanted` -- min max sum avg\n* `gas_used` -- min max sum avg\n* `timeout_height` -- min max avg\n* `events_count` -- min max sum avg\n* `messages_count` -- min max sum avg\n* `fee` -- min max sum avg\n\n#### Event\n* `height` -- min max\n* `time` -- min max\n\n#### Message\n* `height` -- min max\n* `time` -- min max", @@ -2515,6 +2600,14 @@ "type": "string", "example": "28347628346" }, + "gas_limit": { + "type": "integer", + "example": 1234 + }, + "gas_used": { + "type": "integer", + "example": 1234 + }, "inflation_rate": { "type": "string", "example": "0.0800000" @@ -2627,51 +2720,6 @@ } } }, - "responses.GasPriceCandle": { - "type": "object", - "properties": { - "avg_gas_efficiency": { - "type": "string", - "format": "string", - "example": "0.45282" - }, - "avg_gas_price": { - "type": "string", - "format": "string", - "example": "0.45282" - }, - "fee": { - "type": "number", - "format": "integer", - "example": 1283518 - }, - "high": { - "type": "string", - "format": "string", - "example": "0.17632" - }, - "low": { - "type": "string", - "format": "string", - "example": "0.11882" - }, - "time": { - "type": "string", - "format": "date-time", - "example": "2023-07-04T03:10:57+00:00" - }, - "total_gas_limit": { - "type": "string", - "format": "string", - "example": "1213134" - }, - "total_gas_used": { - "type": "string", - "format": "string", - "example": "0.45282" - } - } - }, "responses.HistogramItem": { "type": "object", "properties": { @@ -2835,12 +2883,52 @@ } } }, + "responses.NamespaceUsage": { + "type": "object", + "properties": { + "name": { + "type": "string", + "format": "string", + "example": "00112233" + }, + "size": { + "type": "number", + "format": "integer", + "example": 1283518 + } + } + }, "responses.Params": { "type": "object", "additionalProperties": { "type": "string" } }, + "responses.SeriesItem": { + "type": "object", + "properties": { + "max": { + "type": "string", + "format": "string", + "example": "0.17632" + }, + "min": { + "type": "string", + "format": "string", + "example": "0.17632" + }, + "time": { + "type": "string", + "format": "date-time", + "example": "2023-07-04T03:10:57+00:00" + }, + "value": { + "type": "string", + "format": "string", + "example": "0.17632" + } + } + }, "responses.State": { "type": "object", "properties": { diff --git a/cmd/api/docs/swagger.yaml b/cmd/api/docs/swagger.yaml index f150fd18..4cf010c6 100644 --- a/cmd/api/docs/swagger.yaml +++ b/cmd/api/docs/swagger.yaml @@ -148,6 +148,12 @@ definitions: fee: example: "28347628346" type: string + gas_limit: + example: 1234 + type: integer + gas_used: + example: 1234 + type: integer inflation_rate: example: "0.0800000" type: string @@ -227,41 +233,6 @@ definitions: - $ref: '#/definitions/types.EventType' example: commission type: object - responses.GasPriceCandle: - properties: - avg_gas_efficiency: - example: "0.45282" - format: string - type: string - avg_gas_price: - example: "0.45282" - format: string - type: string - fee: - example: 1283518 - format: integer - type: number - high: - example: "0.17632" - format: string - type: string - low: - example: "0.11882" - format: string - type: string - time: - example: "2023-07-04T03:10:57+00:00" - format: date-time - type: string - total_gas_limit: - example: "1213134" - format: string - type: string - total_gas_used: - example: "0.45282" - format: string - type: string - type: object responses.HistogramItem: properties: time: @@ -386,10 +357,40 @@ definitions: format: string type: string type: object + responses.NamespaceUsage: + properties: + name: + example: "00112233" + format: string + type: string + size: + example: 1283518 + format: integer + type: number + type: object responses.Params: additionalProperties: type: string type: object + responses.SeriesItem: + properties: + max: + example: "0.17632" + format: string + type: string + min: + example: "0.17632" + format: string + type: string + time: + example: "2023-07-04T03:10:57+00:00" + format: date-time + type: string + value: + example: "0.17632" + format: string + type: string + type: object responses.State: properties: hash: @@ -807,11 +808,11 @@ definitions: - MsgTimeout - MsgTimeoutOnClose - MsgAcknowledgement -host: api.celestia.dipdup.net +host: api.celenium.io info: contact: {} - description: This is docs of Celestia indexer API. - title: Swagger Celestia Indexer API + description: This is docs of Celenium API. + title: Swagger Celenium API version: "1.0" paths: /v1/address: @@ -1950,27 +1951,6 @@ paths: summary: Search by hash tags: - search - /v1/stats/gas_price/hourly: - get: - description: Get candles for gas price with volume, average gas efficiency and - fee for an hour - operationId: stats-gas-price-hourly - produces: - - application/json - responses: - "200": - description: OK - schema: - items: - $ref: '#/definitions/responses.GasPriceCandle' - type: array - "500": - description: Internal Server Error - schema: - $ref: '#/definitions/handler.Error' - summary: Get candles for gas price - tags: - - stats /v1/stats/histogram/{table}/{function}/{timeframe}: get: description: |- @@ -2056,6 +2036,89 @@ paths: summary: Get histogram tags: - stats + /v1/stats/namespace/usage: + get: + description: Get namespaces with sorting by size. Returns top 100 namespaces. + Namespaces which is not included to top 100 grouped into 'others' item + operationId: stats-namespace-usage + produces: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/responses.NamespaceUsage' + type: array + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/handler.Error' + summary: Get namespaces with sorting by size. + tags: + - stats + /v1/stats/series/{name}/{timeframe}: + get: + description: Get histogram with precomputed stats by series name and timeframe + operationId: stats-series + parameters: + - description: Timeframe + enum: + - hour + - day + - week + - month + - year + in: path + name: timeframe + required: true + type: string + - description: Series name + enum: + - blobs_size + - tps + - bps + - fee + - supply_change + - block_time + - tx_count + - events_count + - gas_price + - gas_efficiency + - gas_used + - gas_limit + in: path + name: name + required: true + type: string + - description: Time from in unix timestamp + in: query + name: from + type: integer + - description: Time to in unix timestamp + in: query + name: to + type: integer + produces: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/responses.SeriesItem' + type: array + "400": + description: Bad Request + schema: + $ref: '#/definitions/handler.Error' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/handler.Error' + summary: Get histogram with precomputed stats + tags: + - stats /v1/stats/summary/{table}/{function}: get: description: |- diff --git a/cmd/api/handler/responses/block.go b/cmd/api/handler/responses/block.go index 037f5aed..f14cecc9 100644 --- a/cmd/api/handler/responses/block.go +++ b/cmd/api/handler/responses/block.go @@ -74,6 +74,8 @@ type BlockStats struct { SupplyChange string `example:"8635234" json:"supply_change" swaggertype:"string"` InflationRate string `example:"0.0800000" json:"inflation_rate" swaggertype:"string"` BlockTime uint64 `example:"12354" json:"block_time" swaggertype:"integer"` + GasLimit int64 `example:"1234" json:"gas_limit" swaggertype:"integer"` + GasUsed int64 `example:"1234" json:"gas_used" swaggertype:"integer"` MessagesCounts map[types.MsgType]int64 `example:"{MsgPayForBlobs:10,MsgUnjail:1}" json:"messages_counts" swaggertype:"string"` } @@ -87,5 +89,7 @@ func NewBlockStats(stats storage.BlockStats) *BlockStats { InflationRate: stats.InflationRate.String(), BlockTime: stats.BlockTime, MessagesCounts: stats.MessagesCounts, + GasLimit: stats.GasLimit, + GasUsed: stats.GasUsed, } } diff --git a/cmd/api/handler/responses/stats.go b/cmd/api/handler/responses/stats.go index 8f3ba42b..6deaa5e6 100644 --- a/cmd/api/handler/responses/stats.go +++ b/cmd/api/handler/responses/stats.go @@ -1,7 +1,6 @@ package responses import ( - "strconv" "time" "github.com/celenium-io/celestia-indexer/internal/storage" @@ -37,34 +36,6 @@ func NewTxCountHistogramItem(item storage.TxCountForLast24hItem) TxCountHistogra } } -type GasPriceCandle struct { - High string `example:"0.17632" format:"string" json:"high" swaggertype:"string"` - Low string `example:"0.11882" format:"string" json:"low" swaggertype:"string"` - TotalGasLimit string `example:"1213134" format:"string" json:"total_gas_limit" swaggertype:"string"` - TotalGasUsed string `example:"0.45282" format:"string" json:"total_gas_used" swaggertype:"string"` - Fee int64 `example:"1283518" format:"integer" json:"fee" swaggertype:"number"` - GasEfficiency string `example:"0.45282" format:"string" json:"avg_gas_efficiency" swaggertype:"string"` - AvgGasPrice string `example:"0.45282" format:"string" json:"avg_gas_price" swaggertype:"string"` - Time time.Time `example:"2023-07-04T03:10:57+00:00" format:"date-time" json:"time" swaggertype:"string"` -} - -func NewGasPriceCandle(item storage.GasCandle) GasPriceCandle { - return GasPriceCandle{ - Time: item.Time, - High: formatFoat64(item.High), - Low: formatFoat64(item.Low), - Fee: item.Fee, - TotalGasLimit: formatFoat64(item.Volume), - TotalGasUsed: formatFoat64(float64(item.GasUsed)), - GasEfficiency: formatFoat64(float64(item.GasUsed) / item.Volume), - AvgGasPrice: formatFoat64(float64(item.Fee) / item.Volume), - } -} - -func formatFoat64(value float64) string { - return strconv.FormatFloat(value, 'f', -1, 64) -} - type NamespaceUsage struct { Name string `example:"00112233" format:"string" json:"name" swaggertype:"string"` Size int64 `example:"1283518" format:"integer" json:"size" swaggertype:"number"` @@ -76,3 +47,19 @@ func NewNamespaceUsage(ns storage.Namespace) NamespaceUsage { Size: ns.Size, } } + +type SeriesItem struct { + Time time.Time `example:"2023-07-04T03:10:57+00:00" format:"date-time" json:"time" swaggertype:"string"` + Value string `example:"0.17632" format:"string" json:"value" swaggertype:"string"` + Max string `example:"0.17632" format:"string" json:"max,omitempty" swaggertype:"string"` + Min string `example:"0.17632" format:"string" json:"min,omitempty" swaggertype:"string"` +} + +func NewSeriesItem(item storage.SeriesItem) SeriesItem { + return SeriesItem{ + Time: item.Time, + Value: item.Value, + Max: item.Max, + Min: item.Min, + } +} diff --git a/cmd/api/handler/responses/stats_test.go b/cmd/api/handler/responses/stats_test.go deleted file mode 100644 index 6e5612dd..00000000 --- a/cmd/api/handler/responses/stats_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package responses - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func Test_formatFoat64(t *testing.T) { - tests := []struct { - name string - value float64 - want string - }{ - { - name: "test 1", - value: .0123456, - want: "0.0123456", - }, { - name: "test 2", - value: 1234567.0123456, - want: "1234567.0123456", - }, { - name: "test 3", - value: 1234567.0123456789, - want: "1234567.0123456789", - }, { - name: "test 4", - value: -1234567.0123456789, - want: "-1234567.0123456789", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := formatFoat64(tt.value) - require.Equal(t, tt.want, got) - }) - } -} diff --git a/cmd/api/handler/stats.go b/cmd/api/handler/stats.go index 30588ddb..5397960d 100644 --- a/cmd/api/handler/stats.go +++ b/cmd/api/handler/stats.go @@ -191,29 +191,6 @@ func (sh StatsHandler) TxCountHourly24h(c echo.Context) error { return returnArray(c, response) } -// GasPriceHourly godoc -// -// @Summary Get candles for gas price -// @Description Get candles for gas price with volume, average gas efficiency and fee for an hour -// @Tags stats -// @ID stats-gas-price-hourly -// @Produce json -// @Success 200 {array} responses.GasPriceCandle -// @Failure 500 {object} Error -// @Router /v1/stats/gas_price/hourly [get] -func (sh StatsHandler) GasPriceHourly(c echo.Context) error { - histogram, err := sh.repo.GasPriceHourly(c.Request().Context()) - if err != nil { - return internalServerError(c, err) - } - - response := make([]responses.GasPriceCandle, len(histogram)) - for i := range histogram { - response[i] = responses.NewGasPriceCandle(histogram[i]) - } - return returnArray(c, response) -} - // NamespaceUsage godoc // // @Summary Get namespaces with sorting by size. @@ -252,3 +229,46 @@ func (sh StatsHandler) NamespaceUsage(c echo.Context) error { return returnArray(c, response) } + +type seriesRequest struct { + Timeframe string `example:"hour" param:"timeframe" swaggertype:"string" validate:"required,oneof=hour day week month year"` + SeriesName string `example:"tps" param:"name" swaggertype:"string" validate:"required,oneof=blobs_size tps bps fee supply_change block_time tx_count events_count gas_price gas_efficiency gas_used gas_limit"` + From uint64 `example:"1692892095" query:"from" swaggertype:"integer" validate:"omitempty,min=1"` + To uint64 `example:"1692892095" query:"to" swaggertype:"integer" validate:"omitempty,min=1"` +} + +// Series godoc +// +// @Summary Get histogram with precomputed stats +// @Description Get histogram with precomputed stats by series name and timeframe +// @Tags stats +// @ID stats-series +// @Param timeframe path string true "Timeframe" Enums(hour, day, week, month, year) +// @Param name path string true "Series name" Enums(blobs_size, tps, bps, fee, supply_change, block_time, tx_count, events_count, gas_price, gas_efficiency, gas_used, gas_limit) +// @Param from query integer false "Time from in unix timestamp" mininum(1) +// @Param to query integer false "Time to in unix timestamp" mininum(1) +// @Produce json +// @Success 200 {array} responses.SeriesItem +// @Failure 400 {object} Error +// @Failure 500 {object} Error +// @Router /v1/stats/series/{name}/{timeframe} [get] +func (sh StatsHandler) Series(c echo.Context) error { + req, err := bindAndValidate[seriesRequest](c) + if err != nil { + return badRequestError(c, err) + } + + histogram, err := sh.repo.Series(c.Request().Context(), storage.Timeframe(req.Timeframe), req.SeriesName, storage.SeriesRequest{ + From: req.From, + To: req.To, + }) + if err != nil { + return internalServerError(c, err) + } + + response := make([]responses.SeriesItem, len(histogram)) + for i := range histogram { + response[i] = responses.NewSeriesItem(histogram[i]) + } + return returnArray(c, response) +} diff --git a/cmd/api/handler/stats_test.go b/cmd/api/handler/stats_test.go index ef662d29..db4b892c 100644 --- a/cmd/api/handler/stats_test.go +++ b/cmd/api/handler/stats_test.go @@ -264,44 +264,6 @@ func (s *StatsTestSuite) TestTxCount24h() { s.Require().True(testTime.Equal(item.Time)) } -func (s *StatsTestSuite) TestGasPriceHourly() { - req := httptest.NewRequest(http.MethodGet, "/", nil) - rec := httptest.NewRecorder() - c := s.echo.NewContext(req, rec) - c.SetPath("/v1/stats/gas_price/hourly") - - s.stats.EXPECT(). - GasPriceHourly(gomock.Any()). - Return([]storage.GasCandle{ - { - Time: testTime, - High: 1, - Low: .0001, - Volume: 123400, - GasUsed: 13761, - Fee: 1267351, - }, - }, nil) - - s.Require().NoError(s.handler.GasPriceHourly(c)) - s.Require().Equal(http.StatusOK, rec.Code) - - var response []responses.GasPriceCandle - err := json.NewDecoder(rec.Body).Decode(&response) - s.Require().NoError(err) - s.Require().Len(response, 1) - - item := response[0] - s.Require().EqualValues("1", item.High) - s.Require().EqualValues("0.0001", item.Low) - s.Require().EqualValues("123400", item.TotalGasLimit) - s.Require().EqualValues("13761", item.TotalGasUsed) - s.Require().EqualValues(1267351, item.Fee) - s.Require().EqualValues("10.270267423014587", item.AvgGasPrice) - s.Require().EqualValues("0.11151539708265802", item.GasEfficiency) - s.Require().True(testTime.Equal(item.Time)) -} - func (s *StatsTestSuite) TestNamespaceUsage() { req := httptest.NewRequest(http.MethodGet, "/", nil) rec := httptest.NewRecorder() @@ -336,3 +298,60 @@ func (s *StatsTestSuite) TestNamespaceUsage() { s.Require().Equal("others", item1.Name) s.Require().EqualValues(900, item1.Size) } + +func (s *StatsTestSuite) TestBlockStatsHistogram() { + for _, name := range []string{ + storage.SeriesBPS, + storage.SeriesBlobsSize, + storage.SeriesBlockTime, + storage.SeriesEventsCount, + storage.SeriesFee, + storage.SeriesSupplyChange, + storage.SeriesTPS, + storage.SeriesTxCount, + storage.SeriesGasEfficiency, + storage.SeriesGasLimit, + storage.SeriesGasPrice, + storage.SeriesGasUsed, + } { + + for _, tf := range []storage.Timeframe{ + storage.TimeframeHour, + storage.TimeframeDay, + storage.TimeframeWeek, + storage.TimeframeMonth, + storage.TimeframeYear, + } { + req := httptest.NewRequest(http.MethodGet, "/", nil) + rec := httptest.NewRecorder() + c := s.echo.NewContext(req, rec) + c.SetPath("/v1/stats/series/:name/:timeframe") + c.SetParamNames("name", "timeframe") + c.SetParamValues(name, string(tf)) + + s.stats.EXPECT(). + Series(gomock.Any(), tf, name, gomock.Any()). + Return([]storage.SeriesItem{ + { + Time: testTime, + Value: "11234", + Max: "782634", + Min: "69.6665479793", + }, + }, nil) + + s.Require().NoError(s.handler.Series(c)) + s.Require().Equal(http.StatusOK, rec.Code) + + var response []responses.SeriesItem + err := json.NewDecoder(rec.Body).Decode(&response) + s.Require().NoError(err) + s.Require().Len(response, 1) + + item := response[0] + s.Require().Equal("11234", item.Value) + s.Require().Equal("782634", item.Max) + s.Require().Equal("69.6665479793", item.Min) + } + } +} diff --git a/cmd/api/init.go b/cmd/api/init.go index e327f88c..c7c7201f 100644 --- a/cmd/api/init.go +++ b/cmd/api/init.go @@ -309,14 +309,14 @@ func initHandlers(ctx context.Context, e *echo.Echo, cfg Config, db postgres.Sto stats.GET("/tps", statsHandler.TPS) stats.GET("/tx_count_24h", statsHandler.TxCountHourly24h) - gasPrice := stats.Group("/gas_price") - { - gasPrice.GET("/hourly", statsHandler.GasPriceHourly) - } namespace := stats.Group("/namespace") { namespace.GET("/usage", statsHandler.NamespaceUsage) } + series := stats.Group("/series") + { + series.GET("/:name/:timeframe", statsHandler.Series) + } } gasHandler := handler.NewGasHandler() diff --git a/cmd/api/main.go b/cmd/api/main.go index 0a8101fa..865af546 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -12,17 +12,19 @@ import ( "github.com/pkg/errors" "github.com/spf13/cobra" + + _ "github.com/celenium-io/celestia-indexer/cmd/api/docs" ) var rootCmd = &cobra.Command{ Use: "api", - Short: "DipDup Verticals | Celestia API", + Short: "DipDup Verticals | Celenium API", } -// @title Swagger Celestia Indexer API +// @title Swagger Celenium API // @version 1.0 -// @description This is docs of Celestia indexer API. -// @host api.celestia.dipdup.net +// @description This is docs of Celenium API. +// @host api.celenium.io // // @query.collection.format multi func main() { diff --git a/cmd/indexer/main.go b/cmd/indexer/main.go index 79f66a45..993482cb 100644 --- a/cmd/indexer/main.go +++ b/cmd/indexer/main.go @@ -19,7 +19,7 @@ import ( var rootCmd = &cobra.Command{ Use: "indexer", - Short: "DipDup Verticals | Celestia Indexer", + Short: "DipDup Verticals | Celenium Indexer", } func main() { diff --git a/database/views/00_block_stats_by_minute.sql b/database/views/00_block_stats_by_minute.sql new file mode 100644 index 00000000..1cab8605 --- /dev/null +++ b/database/views/00_block_stats_by_minute.sql @@ -0,0 +1,35 @@ +CREATE MATERIALIZED VIEW IF NOT EXISTS block_stats_by_minute +WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS + select + time_bucket('1 minute'::interval, time) AS ts, + (sum(blobs_size)/60.0) as bps, + max(case when block_time > 0 then blobs_size::float/(block_time/1000.0) else 0 end) as bps_max, + min(case when block_time > 0 then blobs_size::float/(block_time/1000.0) else 0 end) as bps_min, + (sum(tx_count)/60.0) as tps, + max(case when block_time > 0 then tx_count::float/(block_time/1000.0) else 0 end) as tps_max, + min(case when block_time > 0 then tx_count::float/(block_time/1000.0) else 0 end) as tps_min, + avg(block_time) as block_time, + sum(blobs_size) as blobs_size, + sum(tx_count) as tx_count, + sum(events_count) as events_count, + sum(fee) as fee, + sum(supply_change) as supply_change, + sum(gas_limit) as gas_limit, + sum(gas_used) as gas_used, + (case when sum(gas_limit) > 0 then sum(fee) / sum(gas_limit) else 0 end) as gas_price, + (case when sum(gas_limit) > 0 then sum(gas_used) / sum(gas_limit) else 0 end) as gas_efficiency + from block_stats + group by 1 + order by 1 desc; + +SELECT add_continuous_aggregate_policy('block_stats_by_minute', + start_offset => NULL, + end_offset => INTERVAL '1 minute', + schedule_interval => INTERVAL '1 minute', + if_not_exists => true) +WHERE NOT (SELECT EXISTS ( + SELECT FROM + "_timescaledb_catalog".continuous_agg + WHERE continuous_agg.user_view_schema = 'public' AND user_view_name = 'block_stats_by_minute' + ) +); diff --git a/database/views/01_block_stats_by_hour.sql b/database/views/01_block_stats_by_hour.sql new file mode 100644 index 00000000..1d224900 --- /dev/null +++ b/database/views/01_block_stats_by_hour.sql @@ -0,0 +1,35 @@ +CREATE MATERIALIZED VIEW IF NOT EXISTS block_stats_by_hour +WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS + select + time_bucket('1 hour'::interval, bbm.ts) AS ts, + sum(blobs_size)/3600.0 as bps, + max(bps_max) as bps_max, + min(bps_min) as bps_min, + sum(tx_count)/3600.0 as tps, + max(tps_max) as tps_max, + min(tps_min) as tps_min, + avg(block_time) as block_time, + sum(blobs_size) as blobs_size, + sum(tx_count) as tx_count, + sum(events_count) as events_count, + sum(fee) as fee, + sum(supply_change) as supply_change, + sum(gas_limit) as gas_limit, + sum(gas_used) as gas_used, + (case when sum(gas_limit) > 0 then sum(fee) / sum(gas_limit) else 0 end) as gas_price, + (case when sum(gas_limit) > 0 then sum(gas_used) / sum(gas_limit) else 0 end) as gas_efficiency + from block_stats_by_minute as bbm + group by 1 + order by 1 desc; + +SELECT add_continuous_aggregate_policy('block_stats_by_hour', + start_offset => NULL, + end_offset => INTERVAL '1 minute', + schedule_interval => INTERVAL '15 minute', + if_not_exists => true) +WHERE NOT (SELECT EXISTS ( + SELECT FROM + "_timescaledb_catalog".continuous_agg + WHERE user_view_schema = 'public' AND user_view_name = 'block_stats_by_hour' + ) +); diff --git a/database/views/02_block_stats_by_day.sql b/database/views/02_block_stats_by_day.sql new file mode 100644 index 00000000..5ee149b3 --- /dev/null +++ b/database/views/02_block_stats_by_day.sql @@ -0,0 +1,35 @@ +CREATE MATERIALIZED VIEW IF NOT EXISTS block_stats_by_day +WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS + select + time_bucket('1 day'::interval, hour.ts) AS ts, + sum(blobs_size)/86400.0 as bps, + max(bps_max) as bps_max, + min(bps_min) as bps_min, + sum(tx_count)/86400.0 as tps, + max(tps_max) as tps_max, + min(tps_min) as tps_min, + avg(block_time) as block_time, + sum(blobs_size) as blobs_size, + sum(tx_count) as tx_count, + sum(events_count) as events_count, + sum(fee) as fee, + sum(supply_change) as supply_change, + sum(gas_limit) as gas_limit, + sum(gas_used) as gas_used, + (case when sum(gas_limit) > 0 then sum(fee) / sum(gas_limit) else 0 end) as gas_price, + (case when sum(gas_limit) > 0 then sum(gas_used) / sum(gas_limit) else 0 end) as gas_efficiency + from block_stats_by_hour as hour + group by 1 + order by 1 desc; + +SELECT add_continuous_aggregate_policy('block_stats_by_day', + start_offset => NULL, + end_offset => INTERVAL '1 minute', + schedule_interval => INTERVAL '1 hour', + if_not_exists => true) +WHERE NOT (SELECT EXISTS ( + SELECT FROM + "_timescaledb_catalog".continuous_agg + WHERE user_view_schema = 'public' AND user_view_name = 'block_stats_by_day' + ) +); \ No newline at end of file diff --git a/database/views/03_block_stats_by_year.sql b/database/views/03_block_stats_by_year.sql new file mode 100644 index 00000000..9c15459c --- /dev/null +++ b/database/views/03_block_stats_by_year.sql @@ -0,0 +1,35 @@ +CREATE MATERIALIZED VIEW IF NOT EXISTS block_stats_by_year +WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS + select + time_bucket('1 year'::interval, day.ts) AS ts, + sum(blobs_size)/(count(*) * 86400.0) as bps, + max(bps_max) as bps_max, + min(bps_min) as bps_min, + sum(tx_count)/(count(*) * 86400.0) as tps, + max(tps_max) as tps_max, + min(tps_min) as tps_min, + avg(block_time) as block_time, + sum(blobs_size) as blobs_size, + sum(tx_count) as tx_count, + sum(events_count) as events_count, + sum(fee) as fee, + sum(supply_change) as supply_change, + sum(gas_limit) as gas_limit, + sum(gas_used) as gas_used, + (case when sum(gas_limit) > 0 then sum(fee) / sum(gas_limit) else 0 end) as gas_price, + (case when sum(gas_limit) > 0 then sum(gas_used) / sum(gas_limit) else 0 end) as gas_efficiency + from block_stats_by_day as day + group by 1 + order by 1 desc; + +SELECT add_continuous_aggregate_policy('block_stats_by_year', + start_offset => NULL, + end_offset => INTERVAL '1 minute', + schedule_interval => INTERVAL '1 hour', + if_not_exists => true) +WHERE NOT (SELECT EXISTS ( + SELECT FROM + "_timescaledb_catalog".continuous_agg + WHERE user_view_schema = 'public' AND user_view_name = 'block_stats_by_year' + ) +); diff --git a/database/views/04_block_stats_by_month.sql b/database/views/04_block_stats_by_month.sql new file mode 100644 index 00000000..f53ad467 --- /dev/null +++ b/database/views/04_block_stats_by_month.sql @@ -0,0 +1,35 @@ +CREATE MATERIALIZED VIEW IF NOT EXISTS block_stats_by_month +WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS + select + time_bucket('1 month'::interval, day.ts) AS ts, + sum(blobs_size)/(count(*) * 86400.0) as bps, + max(bps_max) as bps_max, + min(bps_min) as bps_min, + sum(tx_count)/(count(*) * 86400.0) as tps, + max(tps_max) as tps_max, + min(tps_min) as tps_min, + avg(block_time) as block_time, + sum(blobs_size) as blobs_size, + sum(tx_count) as tx_count, + sum(events_count) as events_count, + sum(fee) as fee, + sum(supply_change) as supply_change, + sum(gas_limit) as gas_limit, + sum(gas_used) as gas_used, + (case when sum(gas_limit) > 0 then sum(fee) / sum(gas_limit) else 0 end) as gas_price, + (case when sum(gas_limit) > 0 then sum(gas_used) / sum(gas_limit) else 0 end) as gas_efficiency + from block_stats_by_day as day + group by 1 + order by 1 desc; + +SELECT add_continuous_aggregate_policy('block_stats_by_month', + start_offset => NULL, + end_offset => INTERVAL '1 minute', + schedule_interval => INTERVAL '1 hour', + if_not_exists => true) +WHERE NOT (SELECT EXISTS ( + SELECT FROM + "_timescaledb_catalog".continuous_agg + WHERE user_view_schema = 'public' AND user_view_name = 'block_stats_by_month' + ) +); diff --git a/database/views/05_block_stats_by_week.sql b/database/views/05_block_stats_by_week.sql new file mode 100644 index 00000000..ddafe4cc --- /dev/null +++ b/database/views/05_block_stats_by_week.sql @@ -0,0 +1,35 @@ +CREATE MATERIALIZED VIEW IF NOT EXISTS block_stats_by_week +WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS + select + time_bucket('1 week'::interval, day.ts) AS ts, + sum(blobs_size)/(7 * 86400.0) as bps, + max(bps_max) as bps_max, + min(bps_min) as bps_min, + sum(tx_count)/(7 * 86400.0) as tps, + max(tps_max) as tps_max, + min(tps_min) as tps_min, + avg(block_time) as block_time, + sum(blobs_size) as blobs_size, + sum(tx_count) as tx_count, + sum(events_count) as events_count, + sum(fee) as fee, + sum(supply_change) as supply_change, + sum(gas_limit) as gas_limit, + sum(gas_used) as gas_used, + (case when sum(gas_limit) > 0 then sum(fee) / sum(gas_limit) else 0 end) as gas_price, + (case when sum(gas_limit) > 0 then sum(gas_used) / sum(gas_limit) else 0 end) as gas_efficiency + from block_stats_by_day as day + group by 1 + order by 1 desc; + +SELECT add_continuous_aggregate_policy('block_stats_by_week', + start_offset => NULL, + end_offset => INTERVAL '1 minute', + schedule_interval => INTERVAL '1 hour', + if_not_exists => true) +WHERE NOT (SELECT EXISTS ( + SELECT FROM + "_timescaledb_catalog".continuous_agg + WHERE user_view_schema = 'public' AND user_view_name = 'block_stats_by_week' + ) +); diff --git a/database/views/gas_price_candlesticks_hourly.sql b/database/views/gas_price_candlesticks_hourly.sql deleted file mode 100644 index 940c9177..00000000 --- a/database/views/gas_price_candlesticks_hourly.sql +++ /dev/null @@ -1,10 +0,0 @@ -CREATE MATERIALIZED VIEW IF NOT EXISTS gas_price_candlesticks_hourly -WITH (timescaledb.continuous) AS - select - time_bucket('1 hour'::interval, time) AS timestamp, - candlestick_agg("time", fee/gas_wanted, gas_wanted) as value, - sum(fee) as fee, - sum(gas_used) as gas_used - from tx - where gas_wanted > 0 and "status" = 'success' - group by timestamp \ No newline at end of file diff --git a/database/views/tx_count_hourly.sql b/database/views/tx_count_hourly.sql deleted file mode 100644 index 9a1afc57..00000000 --- a/database/views/tx_count_hourly.sql +++ /dev/null @@ -1,8 +0,0 @@ -CREATE MATERIALIZED VIEW if not EXISTS tx_count_hourly -WITH (timescaledb.continuous) AS - select - time_bucket('1 hour'::interval, time) AS timestamp, - sum(tx_count) as tx_count, - sum(tx_count) / 3600.0 as tps - from block_stats - group by timestamp \ No newline at end of file diff --git a/internal/storage/block_stats.go b/internal/storage/block_stats.go index ba3b6e1f..8a0eb23e 100644 --- a/internal/storage/block_stats.go +++ b/internal/storage/block_stats.go @@ -30,6 +30,8 @@ type BlockStats struct { EventsCount int64 `bun:"events_count" comment:"Count of events in begin and end of block" stats:"func:min max sum avg"` BlobsSize int64 `bun:"blobs_size" comment:"Summary blocks size from pay for blob" stats:"func:min max sum avg"` BlockTime uint64 `bun:"block_time" comment:"Time in milliseconds between current and previous block" stats:"func:min max sum avg"` + GasLimit int64 `bun:"gas_limit" comment:"Total gas limit in the block"` + GasUsed int64 `bun:"gas_used" comment:"Total gas used in the block"` SupplyChange decimal.Decimal `bun:",type:numeric" comment:"Change of total supply in the block" stats:"func:min max sum avg"` InflationRate decimal.Decimal `bun:",type:numeric" comment:"Inflation rate" stats:"func:min max avg"` Fee decimal.Decimal `bun:"fee,type:numeric" comment:"Summary block fee" stats:"func:min max sum avg"` diff --git a/internal/storage/mock/stats.go b/internal/storage/mock/stats.go index efedb587..36cd1028 100644 --- a/internal/storage/mock/stats.go +++ b/internal/storage/mock/stats.go @@ -78,119 +78,119 @@ func (c *IStatsCountCall) DoAndReturn(f func(context.Context, storage.CountReque return c } -// GasPriceHourly mocks base method. -func (m *MockIStats) GasPriceHourly(ctx context.Context) ([]storage.GasCandle, error) { +// Histogram mocks base method. +func (m *MockIStats) Histogram(ctx context.Context, req storage.HistogramRequest) ([]storage.HistogramItem, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GasPriceHourly", ctx) - ret0, _ := ret[0].([]storage.GasCandle) + ret := m.ctrl.Call(m, "Histogram", ctx, req) + ret0, _ := ret[0].([]storage.HistogramItem) ret1, _ := ret[1].(error) return ret0, ret1 } -// GasPriceHourly indicates an expected call of GasPriceHourly. -func (mr *MockIStatsMockRecorder) GasPriceHourly(ctx any) *IStatsGasPriceHourlyCall { +// Histogram indicates an expected call of Histogram. +func (mr *MockIStatsMockRecorder) Histogram(ctx, req any) *IStatsHistogramCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GasPriceHourly", reflect.TypeOf((*MockIStats)(nil).GasPriceHourly), ctx) - return &IStatsGasPriceHourlyCall{Call: call} + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Histogram", reflect.TypeOf((*MockIStats)(nil).Histogram), ctx, req) + return &IStatsHistogramCall{Call: call} } -// IStatsGasPriceHourlyCall wrap *gomock.Call -type IStatsGasPriceHourlyCall struct { +// IStatsHistogramCall wrap *gomock.Call +type IStatsHistogramCall struct { *gomock.Call } // Return rewrite *gomock.Call.Return -func (c *IStatsGasPriceHourlyCall) Return(arg0 []storage.GasCandle, arg1 error) *IStatsGasPriceHourlyCall { +func (c *IStatsHistogramCall) Return(arg0 []storage.HistogramItem, arg1 error) *IStatsHistogramCall { c.Call = c.Call.Return(arg0, arg1) return c } // Do rewrite *gomock.Call.Do -func (c *IStatsGasPriceHourlyCall) Do(f func(context.Context) ([]storage.GasCandle, error)) *IStatsGasPriceHourlyCall { +func (c *IStatsHistogramCall) Do(f func(context.Context, storage.HistogramRequest) ([]storage.HistogramItem, error)) *IStatsHistogramCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *IStatsGasPriceHourlyCall) DoAndReturn(f func(context.Context) ([]storage.GasCandle, error)) *IStatsGasPriceHourlyCall { +func (c *IStatsHistogramCall) DoAndReturn(f func(context.Context, storage.HistogramRequest) ([]storage.HistogramItem, error)) *IStatsHistogramCall { c.Call = c.Call.DoAndReturn(f) return c } -// Histogram mocks base method. -func (m *MockIStats) Histogram(ctx context.Context, req storage.HistogramRequest) ([]storage.HistogramItem, error) { +// HistogramCount mocks base method. +func (m *MockIStats) HistogramCount(ctx context.Context, req storage.HistogramCountRequest) ([]storage.HistogramItem, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Histogram", ctx, req) + ret := m.ctrl.Call(m, "HistogramCount", ctx, req) ret0, _ := ret[0].([]storage.HistogramItem) ret1, _ := ret[1].(error) return ret0, ret1 } -// Histogram indicates an expected call of Histogram. -func (mr *MockIStatsMockRecorder) Histogram(ctx, req any) *IStatsHistogramCall { +// HistogramCount indicates an expected call of HistogramCount. +func (mr *MockIStatsMockRecorder) HistogramCount(ctx, req any) *IStatsHistogramCountCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Histogram", reflect.TypeOf((*MockIStats)(nil).Histogram), ctx, req) - return &IStatsHistogramCall{Call: call} + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HistogramCount", reflect.TypeOf((*MockIStats)(nil).HistogramCount), ctx, req) + return &IStatsHistogramCountCall{Call: call} } -// IStatsHistogramCall wrap *gomock.Call -type IStatsHistogramCall struct { +// IStatsHistogramCountCall wrap *gomock.Call +type IStatsHistogramCountCall struct { *gomock.Call } // Return rewrite *gomock.Call.Return -func (c *IStatsHistogramCall) Return(arg0 []storage.HistogramItem, arg1 error) *IStatsHistogramCall { +func (c *IStatsHistogramCountCall) Return(arg0 []storage.HistogramItem, arg1 error) *IStatsHistogramCountCall { c.Call = c.Call.Return(arg0, arg1) return c } // Do rewrite *gomock.Call.Do -func (c *IStatsHistogramCall) Do(f func(context.Context, storage.HistogramRequest) ([]storage.HistogramItem, error)) *IStatsHistogramCall { +func (c *IStatsHistogramCountCall) Do(f func(context.Context, storage.HistogramCountRequest) ([]storage.HistogramItem, error)) *IStatsHistogramCountCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *IStatsHistogramCall) DoAndReturn(f func(context.Context, storage.HistogramRequest) ([]storage.HistogramItem, error)) *IStatsHistogramCall { +func (c *IStatsHistogramCountCall) DoAndReturn(f func(context.Context, storage.HistogramCountRequest) ([]storage.HistogramItem, error)) *IStatsHistogramCountCall { c.Call = c.Call.DoAndReturn(f) return c } -// HistogramCount mocks base method. -func (m *MockIStats) HistogramCount(ctx context.Context, req storage.HistogramCountRequest) ([]storage.HistogramItem, error) { +// Series mocks base method. +func (m *MockIStats) Series(ctx context.Context, timeframe storage.Timeframe, name string, req storage.SeriesRequest) ([]storage.SeriesItem, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "HistogramCount", ctx, req) - ret0, _ := ret[0].([]storage.HistogramItem) + ret := m.ctrl.Call(m, "Series", ctx, timeframe, name, req) + ret0, _ := ret[0].([]storage.SeriesItem) ret1, _ := ret[1].(error) return ret0, ret1 } -// HistogramCount indicates an expected call of HistogramCount. -func (mr *MockIStatsMockRecorder) HistogramCount(ctx, req any) *IStatsHistogramCountCall { +// Series indicates an expected call of Series. +func (mr *MockIStatsMockRecorder) Series(ctx, timeframe, name, req any) *IStatsSeriesCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HistogramCount", reflect.TypeOf((*MockIStats)(nil).HistogramCount), ctx, req) - return &IStatsHistogramCountCall{Call: call} + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Series", reflect.TypeOf((*MockIStats)(nil).Series), ctx, timeframe, name, req) + return &IStatsSeriesCall{Call: call} } -// IStatsHistogramCountCall wrap *gomock.Call -type IStatsHistogramCountCall struct { +// IStatsSeriesCall wrap *gomock.Call +type IStatsSeriesCall struct { *gomock.Call } // Return rewrite *gomock.Call.Return -func (c *IStatsHistogramCountCall) Return(arg0 []storage.HistogramItem, arg1 error) *IStatsHistogramCountCall { +func (c *IStatsSeriesCall) Return(arg0 []storage.SeriesItem, arg1 error) *IStatsSeriesCall { c.Call = c.Call.Return(arg0, arg1) return c } // Do rewrite *gomock.Call.Do -func (c *IStatsHistogramCountCall) Do(f func(context.Context, storage.HistogramCountRequest) ([]storage.HistogramItem, error)) *IStatsHistogramCountCall { +func (c *IStatsSeriesCall) Do(f func(context.Context, storage.Timeframe, string, storage.SeriesRequest) ([]storage.SeriesItem, error)) *IStatsSeriesCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *IStatsHistogramCountCall) DoAndReturn(f func(context.Context, storage.HistogramCountRequest) ([]storage.HistogramItem, error)) *IStatsHistogramCountCall { +func (c *IStatsSeriesCall) DoAndReturn(f func(context.Context, storage.Timeframe, string, storage.SeriesRequest) ([]storage.SeriesItem, error)) *IStatsSeriesCall { c.Call = c.Call.DoAndReturn(f) return c } diff --git a/internal/storage/postgres/stats.go b/internal/storage/postgres/stats.go index e93a5601..6ced85e8 100644 --- a/internal/storage/postgres/stats.go +++ b/internal/storage/postgres/stats.go @@ -8,6 +8,7 @@ import ( "github.com/celenium-io/celestia-indexer/internal/storage" "github.com/dipdup-net/go-lib/database" + "github.com/pkg/errors" "github.com/uptrace/bun" ) @@ -110,10 +111,10 @@ func (s Stats) Histogram(ctx context.Context, req storage.HistogramRequest) (res } func (s Stats) TPS(ctx context.Context) (response storage.TPS, err error) { - if err = s.db.DB().NewSelect().Table(storage.ViewTxCountHourly). + if err = s.db.DB().NewSelect().Table(storage.ViewBlockStatsByHour). ColumnExpr("max(tps) as high, min(tps) as low"). - Where("timestamp > date_trunc('hour', now()) - '1 week'::interval"). - Where("timestamp < date_trunc('hour', now())"). + Where("ts > date_trunc('hour', now()) - '1 week'::interval"). + Where("ts < date_trunc('hour', now())"). Scan(ctx, &response.High, &response.Low); err != nil { return } @@ -146,18 +147,68 @@ func (s Stats) TPS(ctx context.Context) (response storage.TPS, err error) { } func (s Stats) TxCountForLast24h(ctx context.Context) (response []storage.TxCountForLast24hItem, err error) { - err = s.db.DB().NewSelect().Table(storage.ViewTxCountHourly). - Where("timestamp > date_trunc('hour', now()) - '25 hours'::interval"). - Where("timestamp < date_trunc('hour', now())"). + err = s.db.DB().NewSelect().Table(storage.ViewBlockStatsByHour). + Column("ts", "tps", "tx_count"). + Where("ts = date_trunc('hour', now()) - '1 day'::interval"). Scan(ctx, &response) return } -func (s Stats) GasPriceHourly(ctx context.Context) (response []storage.GasCandle, err error) { - err = s.db.DB().NewSelect().Table(storage.ViewGasPriceCandlesticksHourly). - ColumnExpr(`high(value) as high, low(value) as low, timestamp, volume(value) as volume, fee, gas_used`). - Where("timestamp > date_trunc('hour', now()) - '1 week'::interval"). - Order("timestamp desc"). - Scan(ctx, &response) +func (s Stats) Series(ctx context.Context, timeframe storage.Timeframe, name string, req storage.SeriesRequest) (response []storage.SeriesItem, err error) { + var view string + switch timeframe { + case storage.TimeframeHour: + view = storage.ViewBlockStatsByHour + case storage.TimeframeDay: + view = storage.ViewBlockStatsByDay + case storage.TimeframeWeek: + view = storage.ViewBlockStatsByWeek + case storage.TimeframeMonth: + view = storage.ViewBlockStatsByMonth + case storage.TimeframeYear: + view = storage.ViewBlockStatsByYear + default: + return nil, errors.Errorf("unexpected timeframe %s", timeframe) + } + + query := s.db.DB().NewSelect().Table(view) + + switch name { + case storage.SeriesBlobsSize: + query.ColumnExpr("ts, blobs_size as value") + case storage.SeriesTPS: + query.ColumnExpr("ts, tps as value, tps_max as max, tps_min as min") + case storage.SeriesBPS: + query.ColumnExpr("ts, bps as value, bps_max as max, bps_min as min") + case storage.SeriesFee: + query.ColumnExpr("ts, fee as value") + case storage.SeriesSupplyChange: + query.ColumnExpr("ts, supply_change as value") + case storage.SeriesBlockTime: + query.ColumnExpr("ts, block_time as value") + case storage.SeriesTxCount: + query.ColumnExpr("ts, tx_count as value") + case storage.SeriesEventsCount: + query.ColumnExpr("ts, events_count as value") + case storage.SeriesGasPrice: + query.ColumnExpr("ts, gas_price as value") + case storage.SeriesGasEfficiency: + query.ColumnExpr("ts, gas_efficiency as value") + case storage.SeriesGasLimit: + query.ColumnExpr("ts, gas_limit as value") + case storage.SeriesGasUsed: + query.ColumnExpr("ts, gas_used as value") + default: + return nil, errors.Errorf("unexpected series name: %s", name) + } + + if req.From > 0 { + query = query.Where("time >= to_timestamp(?)", req.From) + } + if req.To > 0 { + query = query.Where("time < to_timestamp(?)", req.To) + } + + err = query.Limit(100).Scan(ctx, &response) return } diff --git a/internal/storage/postgres/stats_test.go b/internal/storage/postgres/stats_test.go index 64234d7c..2296af09 100644 --- a/internal/storage/postgres/stats_test.go +++ b/internal/storage/postgres/stats_test.go @@ -696,11 +696,11 @@ func (s *StatsTestSuite) TestTxCountForLast24h() { s.Require().Len(items, 0) } -func (s *StatsTestSuite) TestGasPriceHourly() { +func (s *StatsTestSuite) TestSeries() { ctx, ctxCancel := context.WithTimeout(context.Background(), 5*time.Second) defer ctxCancel() - items, err := s.storage.Stats.GasPriceHourly(ctx) + items, err := s.storage.Stats.Series(ctx, storage.TimeframeHour, storage.SeriesBlobsSize, storage.SeriesRequest{}) s.Require().NoError(err) s.Require().Len(items, 0) } diff --git a/internal/storage/postgres/storage_test.go b/internal/storage/postgres/storage_test.go index ba9cce19..397c30d5 100644 --- a/internal/storage/postgres/storage_test.go +++ b/internal/storage/postgres/storage_test.go @@ -150,7 +150,7 @@ func (s *StorageTestSuite) TestBlockByHeightWithStats() { Height: 1000, TxCount: 0, EventsCount: 0, - BlobsSize: 0, + BlobsSize: 1234, BlockTime: 11000, SupplyChange: decimal.NewFromInt(30930476), InflationRate: decimal.NewFromFloat(0.08), @@ -193,7 +193,7 @@ func (s *StorageTestSuite) TestBlockByIdWithRelations() { Height: 1000, TxCount: 0, EventsCount: 0, - BlobsSize: 0, + BlobsSize: 1234, BlockTime: 11000, SupplyChange: decimal.NewFromInt(30930476), InflationRate: decimal.NewFromFloat(0.08), diff --git a/internal/storage/postgres/views.go b/internal/storage/postgres/views.go index 1df5d50f..2fc96764 100644 --- a/internal/storage/postgres/views.go +++ b/internal/storage/postgres/views.go @@ -1,6 +1,7 @@ package postgres import ( + "bytes" "context" "os" "path/filepath" @@ -26,9 +27,21 @@ func (s Storage) createViews(ctx context.Context, conn *database.Bun) error { return err } - if _, err := s.Connection().DB().NewRaw(string(raw)).Exec(ctx); err != nil { - return errors.Wrapf(err, "creating view '%s'", files[i].Name()) + queries := bytes.Split(raw, []byte{';'}) + if len(queries) == 0 { + continue + } + + for _, query := range queries { + query = bytes.TrimLeft(query, "\n ") + if len(query) == 0 { + continue + } + if _, err := s.Connection().DB().NewRaw(string(query)).Exec(ctx); err != nil { + return errors.Wrapf(err, "creating view '%s'", files[i].Name()) + } } + } return nil diff --git a/internal/storage/stats.go b/internal/storage/stats.go index 74e7e2c4..498220a2 100644 --- a/internal/storage/stats.go +++ b/internal/storage/stats.go @@ -96,20 +96,38 @@ type TPS struct { } type TxCountForLast24hItem struct { - Time time.Time `bun:"timestamp"` + Time time.Time `bun:"ts"` TxCount int64 `bun:"tx_count"` TPS float64 `bun:"tps"` } -type GasCandle struct { - High float64 `bun:"high"` - Low float64 `bun:"low"` - Volume float64 `bun:"volume"` - GasUsed int64 `bun:"gas_used"` - Fee int64 `bun:"fee"` - Time time.Time `bun:"timestamp"` +type SeriesRequest struct { + From uint64 + To uint64 } +type SeriesItem struct { + Time time.Time `bun:"ts"` + Value string `bun:"value"` + Max string `bun:"max"` + Min string `bun:"min"` +} + +const ( + SeriesBlobsSize = "blobs_size" + SeriesTPS = "tps" + SeriesBPS = "bps" + SeriesFee = "fee" + SeriesSupplyChange = "supply_change" + SeriesBlockTime = "block_time" + SeriesTxCount = "tx_count" + SeriesEventsCount = "events_count" + SeriesGasPrice = "gas_price" + SeriesGasUsed = "gas_used" + SeriesGasLimit = "gas_limit" + SeriesGasEfficiency = "gas_efficiency" +) + //go:generate mockgen -source=$GOFILE -destination=mock/$GOFILE -package=mock -typed type IStats interface { Count(ctx context.Context, req CountRequest) (string, error) @@ -118,5 +136,5 @@ type IStats interface { Histogram(ctx context.Context, req HistogramRequest) ([]HistogramItem, error) TPS(ctx context.Context) (TPS, error) TxCountForLast24h(ctx context.Context) ([]TxCountForLast24hItem, error) - GasPriceHourly(ctx context.Context) ([]GasCandle, error) + Series(ctx context.Context, timeframe Timeframe, name string, req SeriesRequest) ([]SeriesItem, error) } diff --git a/internal/storage/views.go b/internal/storage/views.go index 124fb013..cb1ad5c2 100644 --- a/internal/storage/views.go +++ b/internal/storage/views.go @@ -1,6 +1,10 @@ package storage const ( - ViewTxCountHourly = "tx_count_hourly" - ViewGasPriceCandlesticksHourly = "gas_price_candlesticks_hourly" + ViewBlockStatsByMinute = "block_stats_by_minute" + ViewBlockStatsByHour = "block_stats_by_hour" + ViewBlockStatsByDay = "block_stats_by_day" + ViewBlockStatsByWeek = "block_stats_by_week" + ViewBlockStatsByMonth = "block_stats_by_month" + ViewBlockStatsByYear = "block_stats_by_year" ) diff --git a/pkg/indexer/parser/parse.go b/pkg/indexer/parser/parse.go index 9f63a472..e19aedaf 100644 --- a/pkg/indexer/parser/parse.go +++ b/pkg/indexer/parser/parse.go @@ -71,6 +71,8 @@ func (p *Module) parse(ctx context.Context, b types.BlockData) error { block.Stats.Fee = block.Stats.Fee.Add(tx.Fee) block.MessageTypes.Set(tx.MessageTypes.Bits) block.Stats.BlobsSize += tx.BlobsSize + block.Stats.GasLimit += tx.GasWanted + block.Stats.GasUsed += tx.GasUsed allEvents = append(allEvents, tx.Events...) } diff --git a/test/data/block_stats.yml b/test/data/block_stats.yml index 0bbdb601..0f89edd7 100644 --- a/test/data/block_stats.yml +++ b/test/data/block_stats.yml @@ -7,6 +7,9 @@ supply_change: 30930476 inflation_rate: 0.080000000000000000 block_time: 11000 + blobs_size: 1234 + gas_limit: 160820 + gas_used: 154966 - id: 1 height: 999 @@ -17,3 +20,6 @@ supply_change: 20930476 inflation_rate: 0.080000000000000000 block_time: 11423 + blobs_size: 2412 + gas_limit: 80410 + gas_used: 77483 diff --git a/test/data/rollback/block_stats.yml b/test/data/rollback/block_stats.yml index 2dcf3f0c..cc2733ea 100644 --- a/test/data/rollback/block_stats.yml +++ b/test/data/rollback/block_stats.yml @@ -8,6 +8,8 @@ supply_change: 23590834 inflation_rate: 0 block_time: 11000 + gas_limit: 0 + gas_used: 0 - id: 2 height: 1000 @@ -19,6 +21,8 @@ supply_change: 30930476 inflation_rate: 0.080000000000000000 block_time: 11000 + gas_limit: 0 + gas_used: 0 - id: 1 height: 999 @@ -30,3 +34,5 @@ supply_change: 20930476 inflation_rate: 0.080000000000000000 block_time: 11000 + gas_limit: 0 + gas_used: 0