From f11918165756c5931e40e72f78835409e773b55d Mon Sep 17 00:00:00 2001 From: Artem Date: Thu, 23 Nov 2023 16:08:34 +0100 Subject: [PATCH 1/2] Feature: extended stats endpoint for namespace --- cmd/api/docs/docs.go | 87 ++++++++++++++++++- cmd/api/docs/swagger.json | 84 ++++++++++++++++++ cmd/api/docs/swagger.yaml | 60 +++++++++++++ cmd/api/handler/stats.go | 64 ++++++++++++++ cmd/api/handler/stats_test.go | 49 +++++++++++ cmd/api/init.go | 1 + database/views/06_namespace_stats_by_hour.sql | 23 +++++ database/views/07_namespace_stats_by_day.sql | 23 +++++ database/views/08_namespace_stats_by_year.sql | 23 +++++ .../views/09_namespace_stats_by_month.sql | 23 +++++ database/views/10_namespace_stats_by_week.sql | 23 +++++ internal/storage/mock/address.go | 3 - internal/storage/mock/balance.go | 3 - internal/storage/mock/block.go | 3 - internal/storage/mock/block_stats.go | 3 - internal/storage/mock/constant.go | 3 - internal/storage/mock/denom_metadata.go | 3 - internal/storage/mock/event.go | 3 - internal/storage/mock/generic.go | 3 - internal/storage/mock/message.go | 3 - internal/storage/mock/namespace.go | 3 - internal/storage/mock/state.go | 3 - internal/storage/mock/stats.go | 42 ++++++++- internal/storage/mock/tx.go | 3 - internal/storage/mock/validator.go | 3 - internal/storage/namespace_message.go | 5 +- internal/storage/postgres/core.go | 1 + internal/storage/postgres/stats.go | 39 +++++++++ internal/storage/postgres/stats_test.go | 9 ++ internal/storage/stats.go | 3 + internal/storage/types/event_type_enum.go | 3 - internal/storage/types/module_enum.go | 3 - .../storage/types/msg_address_type_enum.go | 3 - internal/storage/types/msg_type_enum.go | 3 - internal/storage/types/status_enum.go | 3 - internal/storage/views.go | 17 ++-- pkg/indexer/genesis/save.go | 1 + pkg/indexer/storage/message.go | 1 + pkg/node/mock/api.go | 3 - 39 files changed, 564 insertions(+), 71 deletions(-) create mode 100644 database/views/06_namespace_stats_by_hour.sql create mode 100644 database/views/07_namespace_stats_by_day.sql create mode 100644 database/views/08_namespace_stats_by_year.sql create mode 100644 database/views/09_namespace_stats_by_month.sql create mode 100644 database/views/10_namespace_stats_by_week.sql diff --git a/cmd/api/docs/docs.go b/cmd/api/docs/docs.go index e7f4700b..17cc1b72 100644 --- a/cmd/api/docs/docs.go +++ b/cmd/api/docs/docs.go @@ -1,6 +1,3 @@ -// SPDX-FileCopyrightText: 2023 PK Lab AG -// SPDX-License-Identifier: MIT - // Package docs Code generated by swaggo/swag. DO NOT EDIT package docs @@ -1654,6 +1651,90 @@ const docTemplate = `{ } } }, + "/v1/stats/namespace/series/{id}/{name}/{timeframe}": { + "get": { + "description": "Get histogram for namespace with precomputed stats by series name and timeframe", + "produces": [ + "application/json" + ], + "tags": [ + "stats" + ], + "summary": "Get histogram for namespace with precomputed stats", + "operationId": "stats-ns-series", + "parameters": [ + { + "maxLength": 56, + "minLength": 56, + "type": "string", + "description": "Namespace id in hexadecimal", + "name": "id", + "in": "path", + "required": true + }, + { + "enum": [ + "hour", + "day", + "week", + "month", + "year" + ], + "type": "string", + "description": "Timeframe", + "name": "timeframe", + "in": "path", + "required": true + }, + { + "enum": [ + "pfb_count", + "size" + ], + "type": "string", + "description": "Series name", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "integer", + "description": "Time from in unix timestamp", + "name": "from", + "in": "query" + }, + { + "type": "integer", + "description": "Time to in unix timestamp", + "name": "to", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/responses.SeriesItem" + } + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/handler.Error" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handler.Error" + } + } + } + } + }, "/v1/stats/namespace/usage": { "get": { "description": "Get namespaces with sorting by size. Returns top 100 namespaces. Namespaces which is not included to top 100 grouped into 'others' item", diff --git a/cmd/api/docs/swagger.json b/cmd/api/docs/swagger.json index 1ffc4138..68003182 100644 --- a/cmd/api/docs/swagger.json +++ b/cmd/api/docs/swagger.json @@ -1644,6 +1644,90 @@ } } }, + "/v1/stats/namespace/series/{id}/{name}/{timeframe}": { + "get": { + "description": "Get histogram for namespace with precomputed stats by series name and timeframe", + "produces": [ + "application/json" + ], + "tags": [ + "stats" + ], + "summary": "Get histogram for namespace with precomputed stats", + "operationId": "stats-ns-series", + "parameters": [ + { + "maxLength": 56, + "minLength": 56, + "type": "string", + "description": "Namespace id in hexadecimal", + "name": "id", + "in": "path", + "required": true + }, + { + "enum": [ + "hour", + "day", + "week", + "month", + "year" + ], + "type": "string", + "description": "Timeframe", + "name": "timeframe", + "in": "path", + "required": true + }, + { + "enum": [ + "pfb_count", + "size" + ], + "type": "string", + "description": "Series name", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "integer", + "description": "Time from in unix timestamp", + "name": "from", + "in": "query" + }, + { + "type": "integer", + "description": "Time to in unix timestamp", + "name": "to", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/responses.SeriesItem" + } + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/handler.Error" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/handler.Error" + } + } + } + } + }, "/v1/stats/namespace/usage": { "get": { "description": "Get namespaces with sorting by size. Returns top 100 namespaces. Namespaces which is not included to top 100 grouped into 'others' item", diff --git a/cmd/api/docs/swagger.yaml b/cmd/api/docs/swagger.yaml index ab49b98e..67bc291c 100644 --- a/cmd/api/docs/swagger.yaml +++ b/cmd/api/docs/swagger.yaml @@ -2097,6 +2097,66 @@ paths: summary: Get histogram tags: - stats + /v1/stats/namespace/series/{id}/{name}/{timeframe}: + get: + description: Get histogram for namespace with precomputed stats by series name + and timeframe + operationId: stats-ns-series + parameters: + - description: Namespace id in hexadecimal + in: path + maxLength: 56 + minLength: 56 + name: id + required: true + type: string + - description: Timeframe + enum: + - hour + - day + - week + - month + - year + in: path + name: timeframe + required: true + type: string + - description: Series name + enum: + - pfb_count + - size + in: path + name: name + required: true + type: string + - description: Time from in unix timestamp + in: query + name: from + type: integer + - description: Time to in unix timestamp + in: query + name: to + type: integer + produces: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/responses.SeriesItem' + type: array + "400": + description: Bad Request + schema: + $ref: '#/definitions/handler.Error' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/handler.Error' + summary: Get histogram for namespace with precomputed stats + tags: + - stats /v1/stats/namespace/usage: get: description: Get namespaces with sorting by size. Returns top 100 namespaces. diff --git a/cmd/api/handler/stats.go b/cmd/api/handler/stats.go index 5397960d..5e663369 100644 --- a/cmd/api/handler/stats.go +++ b/cmd/api/handler/stats.go @@ -4,6 +4,7 @@ package handler import ( + "encoding/hex" "errors" "net/http" @@ -272,3 +273,66 @@ func (sh StatsHandler) Series(c echo.Context) error { } return returnArray(c, response) } + +type namespaceSeriesRequest struct { + Id string `example:"0011223344" param:"id" swaggertype:"string" validate:"required,hexadecimal,len=56"` + Timeframe string `example:"hour" param:"timeframe" swaggertype:"string" validate:"required,oneof=hour day week month year"` + SeriesName string `example:"size" param:"name" swaggertype:"string" validate:"required,oneof=pfb_count size"` + From uint64 `example:"1692892095" query:"from" swaggertype:"integer" validate:"omitempty,min=1"` + To uint64 `example:"1692892095" query:"to" swaggertype:"integer" validate:"omitempty,min=1"` +} + +// NamespaceSeries godoc +// +// @Summary Get histogram for namespace with precomputed stats +// @Description Get histogram for namespace with precomputed stats by series name and timeframe +// @Tags stats +// @ID stats-ns-series +// @Param id path string true "Namespace id in hexadecimal" minlength(56) maxlength(56) +// @Param timeframe path string true "Timeframe" Enums(hour, day, week, month, year) +// @Param name path string true "Series name" Enums(pfb_count, size) +// @Param from query integer false "Time from in unix timestamp" mininum(1) +// @Param to query integer false "Time to in unix timestamp" mininum(1) +// @Produce json +// @Success 200 {array} responses.SeriesItem +// @Failure 400 {object} Error +// @Failure 500 {object} Error +// @Router /v1/stats/namespace/series/{id}/{name}/{timeframe} [get] +func (sh StatsHandler) NamespaceSeries(c echo.Context) error { + req, err := bindAndValidate[namespaceSeriesRequest](c) + if err != nil { + return badRequestError(c, err) + } + + namespaceId, err := hex.DecodeString(req.Id) + if err != nil { + return badRequestError(c, err) + } + + namespace, err := sh.nsRepo.ByNamespaceId(c.Request().Context(), namespaceId) + if err != nil { + return handleError(c, err, sh.nsRepo) + } + if len(namespace) == 0 { + return c.JSON(http.StatusOK, []any{}) + } + + histogram, err := sh.repo.NamespaceSeries( + c.Request().Context(), + storage.Timeframe(req.Timeframe), + req.SeriesName, + namespace[0].Id, + storage.SeriesRequest{ + From: req.From, + To: req.To, + }) + if err != nil { + return internalServerError(c, err) + } + + response := make([]responses.SeriesItem, len(histogram)) + for i := range histogram { + response[i] = responses.NewSeriesItem(histogram[i]) + } + return returnArray(c, response) +} diff --git a/cmd/api/handler/stats_test.go b/cmd/api/handler/stats_test.go index db4b892c..812772e8 100644 --- a/cmd/api/handler/stats_test.go +++ b/cmd/api/handler/stats_test.go @@ -355,3 +355,52 @@ func (s *StatsTestSuite) TestBlockStatsHistogram() { } } } + +func (s *StatsTestSuite) TestNamespaceStatsHistogram() { + for _, name := range []string{ + storage.SeriesNsPfbCount, + storage.SeriesNsSize, + } { + + for _, tf := range []storage.Timeframe{ + storage.TimeframeHour, + storage.TimeframeDay, + storage.TimeframeWeek, + storage.TimeframeMonth, + storage.TimeframeYear, + } { + req := httptest.NewRequest(http.MethodGet, "/", nil) + rec := httptest.NewRecorder() + c := s.echo.NewContext(req, rec) + c.SetPath("/v1/stats/namespace/series/:id/:name/:timeframe") + c.SetParamNames("id", "name", "timeframe") + c.SetParamValues("000000000000000000000000000000000000000008E5F679BF7116CB", name, string(tf)) + + s.ns.EXPECT(). + ByNamespaceId(gomock.Any(), gomock.Any()). + Return([]storage.Namespace{ + testNamespace, + }, nil) + + s.stats.EXPECT(). + NamespaceSeries(gomock.Any(), tf, name, testNamespace.Id, gomock.Any()). + Return([]storage.SeriesItem{ + { + Time: testTime, + Value: "11234", + }, + }, nil) + + s.Require().NoError(s.handler.NamespaceSeries(c)) + s.Require().Equal(http.StatusOK, rec.Code) + + var response []responses.SeriesItem + err := json.NewDecoder(rec.Body).Decode(&response) + s.Require().NoError(err) + s.Require().Len(response, 1) + + item := response[0] + s.Require().Equal("11234", item.Value) + } + } +} diff --git a/cmd/api/init.go b/cmd/api/init.go index a6c9d97e..5ca8fa87 100644 --- a/cmd/api/init.go +++ b/cmd/api/init.go @@ -324,6 +324,7 @@ func initHandlers(ctx context.Context, e *echo.Echo, cfg Config, db postgres.Sto namespace := stats.Group("/namespace") { namespace.GET("/usage", statsHandler.NamespaceUsage) + namespace.GET("/series/:id/:name/:timeframe", statsHandler.NamespaceSeries) } series := stats.Group("/series") { diff --git a/database/views/06_namespace_stats_by_hour.sql b/database/views/06_namespace_stats_by_hour.sql new file mode 100644 index 00000000..013c722e --- /dev/null +++ b/database/views/06_namespace_stats_by_hour.sql @@ -0,0 +1,23 @@ +CREATE MATERIALIZED VIEW IF NOT EXISTS namespace_stats_by_hour +WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS + select + time_bucket('1 hour'::interval, nm.time) AS ts, + nm.namespace_id, + count(*) as pfb_count, + sum(size) as size + from namespace_message as nm + group by 1, 2 + order by 1 desc; + + +SELECT add_continuous_aggregate_policy('namespace_stats_by_hour', + start_offset => NULL, + end_offset => INTERVAL '1 minute', + schedule_interval => INTERVAL '15 minute', + if_not_exists => true) +WHERE NOT (SELECT EXISTS ( + SELECT FROM + "_timescaledb_catalog".continuous_agg + WHERE user_view_schema = 'public' AND user_view_name = 'namespace_stats_by_hour' + ) +); diff --git a/database/views/07_namespace_stats_by_day.sql b/database/views/07_namespace_stats_by_day.sql new file mode 100644 index 00000000..59c26670 --- /dev/null +++ b/database/views/07_namespace_stats_by_day.sql @@ -0,0 +1,23 @@ +CREATE MATERIALIZED VIEW IF NOT EXISTS namespace_stats_by_day +WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS + select + time_bucket('1 day'::interval, nm.ts) AS ts, + nm.namespace_id, + count(*) as pfb_count, + sum(size) as size + from namespace_stats_by_hour as nm + group by 1, 2 + order by 1 desc; + + +SELECT add_continuous_aggregate_policy('namespace_stats_by_day', + start_offset => NULL, + end_offset => INTERVAL '1 minute', + schedule_interval => INTERVAL '1 hour', + if_not_exists => true) +WHERE NOT (SELECT EXISTS ( + SELECT FROM + "_timescaledb_catalog".continuous_agg + WHERE user_view_schema = 'public' AND user_view_name = 'namespace_stats_by_day' + ) +); diff --git a/database/views/08_namespace_stats_by_year.sql b/database/views/08_namespace_stats_by_year.sql new file mode 100644 index 00000000..6963c050 --- /dev/null +++ b/database/views/08_namespace_stats_by_year.sql @@ -0,0 +1,23 @@ +CREATE MATERIALIZED VIEW IF NOT EXISTS namespace_stats_by_year +WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS + select + time_bucket('1 year'::interval, nm.ts) AS ts, + nm.namespace_id, + count(*) as pfb_count, + sum(size) as size + from namespace_stats_by_day as nm + group by 1, 2 + order by 1 desc; + + +SELECT add_continuous_aggregate_policy('namespace_stats_by_year', + start_offset => NULL, + end_offset => INTERVAL '1 minute', + schedule_interval => INTERVAL '1 hour', + if_not_exists => true) +WHERE NOT (SELECT EXISTS ( + SELECT FROM + "_timescaledb_catalog".continuous_agg + WHERE user_view_schema = 'public' AND user_view_name = 'namespace_stats_by_year' + ) +); diff --git a/database/views/09_namespace_stats_by_month.sql b/database/views/09_namespace_stats_by_month.sql new file mode 100644 index 00000000..886e08d1 --- /dev/null +++ b/database/views/09_namespace_stats_by_month.sql @@ -0,0 +1,23 @@ +CREATE MATERIALIZED VIEW IF NOT EXISTS namespace_stats_by_month + WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS + select + time_bucket('1 month'::interval, nm.ts) AS ts, + nm.namespace_id, + count(*) as pfb_count, + sum(size) as size + from namespace_stats_by_day as nm + group by 1, 2 + order by 1 desc; + + +SELECT add_continuous_aggregate_policy('namespace_stats_by_month', + start_offset => NULL, + end_offset => INTERVAL '1 minute', + schedule_interval => INTERVAL '1 hour', + if_not_exists => true) +WHERE NOT (SELECT EXISTS ( + SELECT FROM + "_timescaledb_catalog".continuous_agg + WHERE continuous_agg.user_view_schema = 'public' AND user_view_name = 'namespace_stats_by_month' + ) +) \ No newline at end of file diff --git a/database/views/10_namespace_stats_by_week.sql b/database/views/10_namespace_stats_by_week.sql new file mode 100644 index 00000000..753a0541 --- /dev/null +++ b/database/views/10_namespace_stats_by_week.sql @@ -0,0 +1,23 @@ +CREATE MATERIALIZED VIEW IF NOT EXISTS namespace_stats_by_week +WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS + select + time_bucket('1 week'::interval, nm.ts) AS ts, + nm.namespace_id, + count(*) as pfb_count, + sum(size) as size + from namespace_stats_by_day as nm + group by 1, 2 + order by 1 desc; + + +SELECT add_continuous_aggregate_policy('namespace_stats_by_week', + start_offset => NULL, + end_offset => INTERVAL '1 minute', + schedule_interval => INTERVAL '1 hour', + if_not_exists => true) +WHERE NOT (SELECT EXISTS ( + SELECT FROM + "_timescaledb_catalog".continuous_agg + WHERE user_view_schema = 'public' AND user_view_name = 'namespace_stats_by_week' + ) +); diff --git a/internal/storage/mock/address.go b/internal/storage/mock/address.go index 8a6f7ff5..7d380590 100644 --- a/internal/storage/mock/address.go +++ b/internal/storage/mock/address.go @@ -1,6 +1,3 @@ -// SPDX-FileCopyrightText: 2023 PK Lab AG -// SPDX-License-Identifier: MIT - // Code generated by MockGen. DO NOT EDIT. // Source: address.go // diff --git a/internal/storage/mock/balance.go b/internal/storage/mock/balance.go index 28bd1ceb..684c1bc2 100644 --- a/internal/storage/mock/balance.go +++ b/internal/storage/mock/balance.go @@ -1,6 +1,3 @@ -// SPDX-FileCopyrightText: 2023 PK Lab AG -// SPDX-License-Identifier: MIT - // Code generated by MockGen. DO NOT EDIT. // Source: balance.go // diff --git a/internal/storage/mock/block.go b/internal/storage/mock/block.go index 24f7f489..b3af1472 100644 --- a/internal/storage/mock/block.go +++ b/internal/storage/mock/block.go @@ -1,6 +1,3 @@ -// SPDX-FileCopyrightText: 2023 PK Lab AG -// SPDX-License-Identifier: MIT - // Code generated by MockGen. DO NOT EDIT. // Source: block.go // diff --git a/internal/storage/mock/block_stats.go b/internal/storage/mock/block_stats.go index acbaf15c..bbf39bbd 100644 --- a/internal/storage/mock/block_stats.go +++ b/internal/storage/mock/block_stats.go @@ -1,6 +1,3 @@ -// SPDX-FileCopyrightText: 2023 PK Lab AG -// SPDX-License-Identifier: MIT - // Code generated by MockGen. DO NOT EDIT. // Source: block_stats.go // diff --git a/internal/storage/mock/constant.go b/internal/storage/mock/constant.go index 0149aa61..ee1891af 100644 --- a/internal/storage/mock/constant.go +++ b/internal/storage/mock/constant.go @@ -1,6 +1,3 @@ -// SPDX-FileCopyrightText: 2023 PK Lab AG -// SPDX-License-Identifier: MIT - // Code generated by MockGen. DO NOT EDIT. // Source: constant.go // diff --git a/internal/storage/mock/denom_metadata.go b/internal/storage/mock/denom_metadata.go index 2f2f30ae..1c9c6e60 100644 --- a/internal/storage/mock/denom_metadata.go +++ b/internal/storage/mock/denom_metadata.go @@ -1,6 +1,3 @@ -// SPDX-FileCopyrightText: 2023 PK Lab AG -// SPDX-License-Identifier: MIT - // Code generated by MockGen. DO NOT EDIT. // Source: denom_metadata.go // diff --git a/internal/storage/mock/event.go b/internal/storage/mock/event.go index 97301152..52cdd576 100644 --- a/internal/storage/mock/event.go +++ b/internal/storage/mock/event.go @@ -1,6 +1,3 @@ -// SPDX-FileCopyrightText: 2023 PK Lab AG -// SPDX-License-Identifier: MIT - // Code generated by MockGen. DO NOT EDIT. // Source: event.go // diff --git a/internal/storage/mock/generic.go b/internal/storage/mock/generic.go index 7f6dd32c..505c9d6e 100644 --- a/internal/storage/mock/generic.go +++ b/internal/storage/mock/generic.go @@ -1,6 +1,3 @@ -// SPDX-FileCopyrightText: 2023 PK Lab AG -// SPDX-License-Identifier: MIT - // Code generated by MockGen. DO NOT EDIT. // Source: generic.go // diff --git a/internal/storage/mock/message.go b/internal/storage/mock/message.go index 4681cd51..02d820a6 100644 --- a/internal/storage/mock/message.go +++ b/internal/storage/mock/message.go @@ -1,6 +1,3 @@ -// SPDX-FileCopyrightText: 2023 PK Lab AG -// SPDX-License-Identifier: MIT - // Code generated by MockGen. DO NOT EDIT. // Source: message.go // diff --git a/internal/storage/mock/namespace.go b/internal/storage/mock/namespace.go index 6679865e..6787b869 100644 --- a/internal/storage/mock/namespace.go +++ b/internal/storage/mock/namespace.go @@ -1,6 +1,3 @@ -// SPDX-FileCopyrightText: 2023 PK Lab AG -// SPDX-License-Identifier: MIT - // Code generated by MockGen. DO NOT EDIT. // Source: namespace.go // diff --git a/internal/storage/mock/state.go b/internal/storage/mock/state.go index 5b7e4fe9..37c546da 100644 --- a/internal/storage/mock/state.go +++ b/internal/storage/mock/state.go @@ -1,6 +1,3 @@ -// SPDX-FileCopyrightText: 2023 PK Lab AG -// SPDX-License-Identifier: MIT - // Code generated by MockGen. DO NOT EDIT. // Source: state.go // diff --git a/internal/storage/mock/stats.go b/internal/storage/mock/stats.go index f55287fb..014b301e 100644 --- a/internal/storage/mock/stats.go +++ b/internal/storage/mock/stats.go @@ -1,6 +1,3 @@ -// SPDX-FileCopyrightText: 2023 PK Lab AG -// SPDX-License-Identifier: MIT - // Code generated by MockGen. DO NOT EDIT. // Source: stats.go // @@ -159,6 +156,45 @@ func (c *IStatsHistogramCountCall) DoAndReturn(f func(context.Context, storage.H return c } +// NamespaceSeries mocks base method. +func (m *MockIStats) NamespaceSeries(ctx context.Context, timeframe storage.Timeframe, name string, nsId uint64, req storage.SeriesRequest) ([]storage.SeriesItem, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NamespaceSeries", ctx, timeframe, name, nsId, req) + ret0, _ := ret[0].([]storage.SeriesItem) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// NamespaceSeries indicates an expected call of NamespaceSeries. +func (mr *MockIStatsMockRecorder) NamespaceSeries(ctx, timeframe, name, nsId, req any) *IStatsNamespaceSeriesCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NamespaceSeries", reflect.TypeOf((*MockIStats)(nil).NamespaceSeries), ctx, timeframe, name, nsId, req) + return &IStatsNamespaceSeriesCall{Call: call} +} + +// IStatsNamespaceSeriesCall wrap *gomock.Call +type IStatsNamespaceSeriesCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *IStatsNamespaceSeriesCall) Return(response []storage.SeriesItem, err error) *IStatsNamespaceSeriesCall { + c.Call = c.Call.Return(response, err) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *IStatsNamespaceSeriesCall) Do(f func(context.Context, storage.Timeframe, string, uint64, storage.SeriesRequest) ([]storage.SeriesItem, error)) *IStatsNamespaceSeriesCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *IStatsNamespaceSeriesCall) DoAndReturn(f func(context.Context, storage.Timeframe, string, uint64, storage.SeriesRequest) ([]storage.SeriesItem, error)) *IStatsNamespaceSeriesCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + // Series mocks base method. func (m *MockIStats) Series(ctx context.Context, timeframe storage.Timeframe, name string, req storage.SeriesRequest) ([]storage.SeriesItem, error) { m.ctrl.T.Helper() diff --git a/internal/storage/mock/tx.go b/internal/storage/mock/tx.go index 8b9c298a..88c02e45 100644 --- a/internal/storage/mock/tx.go +++ b/internal/storage/mock/tx.go @@ -1,6 +1,3 @@ -// SPDX-FileCopyrightText: 2023 PK Lab AG -// SPDX-License-Identifier: MIT - // Code generated by MockGen. DO NOT EDIT. // Source: tx.go // diff --git a/internal/storage/mock/validator.go b/internal/storage/mock/validator.go index 666d5a1c..ceb55561 100644 --- a/internal/storage/mock/validator.go +++ b/internal/storage/mock/validator.go @@ -1,6 +1,3 @@ -// SPDX-FileCopyrightText: 2023 PK Lab AG -// SPDX-License-Identifier: MIT - // Code generated by MockGen. DO NOT EDIT. // Source: validator.go // diff --git a/internal/storage/namespace_message.go b/internal/storage/namespace_message.go index cd36f0ef..75efb58d 100644 --- a/internal/storage/namespace_message.go +++ b/internal/storage/namespace_message.go @@ -18,8 +18,9 @@ type NamespaceMessage struct { MsgId uint64 `bun:"msg_id,pk" comment:"Message id"` TxId uint64 `bun:"tx_id" comment:"Transaction id"` - Time time.Time `bun:"time,notnull" comment:"Message time"` - Height types.Level `bun:"height" comment:"Message block height"` + Time time.Time `bun:"time,notnull,pk" comment:"Message time"` + Height types.Level `bun:"height" comment:"Message block height"` + Size uint64 `bun:"size" comment:"Total namespace size change due to message"` Message *Message `bun:"rel:belongs-to,join:msg_id=id"` Namespace *Namespace `bun:"rel:belongs-to,join:namespace_id=id"` diff --git a/internal/storage/postgres/core.go b/internal/storage/postgres/core.go index 75c955f1..c81c8927 100644 --- a/internal/storage/postgres/core.go +++ b/internal/storage/postgres/core.go @@ -119,6 +119,7 @@ func createHypertables(ctx context.Context, conn *database.Bun) error { &models.Tx{}, &models.Message{}, &models.Event{}, + &models.NamespaceMessage{}, } { if _, err := tx.ExecContext(ctx, `SELECT create_hypertable(?, 'time', chunk_time_interval => INTERVAL '1 month', if_not_exists => TRUE);`, diff --git a/internal/storage/postgres/stats.go b/internal/storage/postgres/stats.go index 6ced85e8..76e23b9f 100644 --- a/internal/storage/postgres/stats.go +++ b/internal/storage/postgres/stats.go @@ -212,3 +212,42 @@ func (s Stats) Series(ctx context.Context, timeframe storage.Timeframe, name str err = query.Limit(100).Scan(ctx, &response) return } + +func (s Stats) NamespaceSeries(ctx context.Context, timeframe storage.Timeframe, name string, nsId uint64, req storage.SeriesRequest) (response []storage.SeriesItem, err error) { + var view string + switch timeframe { + case storage.TimeframeHour: + view = storage.ViewNamespaceStatsByHour + case storage.TimeframeDay: + view = storage.ViewNamespaceStatsByDay + case storage.TimeframeWeek: + view = storage.ViewNamespaceStatsByWeek + case storage.TimeframeMonth: + view = storage.ViewNamespaceStatsByMonth + case storage.TimeframeYear: + view = storage.ViewNamespaceStatsByYear + default: + return nil, errors.Errorf("unexpected timeframe %s", timeframe) + } + + query := s.db.DB().NewSelect().Table(view).Where("namespace_id = ?", nsId) + + switch name { + case storage.SeriesNsPfbCount: + query.ColumnExpr("ts, pfb_count as value") + case storage.SeriesNsSize: + query.ColumnExpr("ts, size as value") + default: + return nil, errors.Errorf("unexpected series name: %s", name) + } + + if req.From > 0 { + query = query.Where("time >= to_timestamp(?)", req.From) + } + if req.To > 0 { + query = query.Where("time < to_timestamp(?)", req.To) + } + + err = query.Limit(100).Scan(ctx, &response) + return +} diff --git a/internal/storage/postgres/stats_test.go b/internal/storage/postgres/stats_test.go index 2296af09..26ac2c0f 100644 --- a/internal/storage/postgres/stats_test.go +++ b/internal/storage/postgres/stats_test.go @@ -705,6 +705,15 @@ func (s *StatsTestSuite) TestSeries() { s.Require().Len(items, 0) } +func (s *StatsTestSuite) TestNamespaceSeries() { + ctx, ctxCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer ctxCancel() + + items, err := s.storage.Stats.NamespaceSeries(ctx, storage.TimeframeHour, storage.SeriesNsSize, 1, storage.SeriesRequest{}) + s.Require().NoError(err) + s.Require().Len(items, 0) +} + func TestSuiteStats_Run(t *testing.T) { suite.Run(t, new(StatsTestSuite)) } diff --git a/internal/storage/stats.go b/internal/storage/stats.go index 498220a2..f22bbae0 100644 --- a/internal/storage/stats.go +++ b/internal/storage/stats.go @@ -126,6 +126,8 @@ const ( SeriesGasUsed = "gas_used" SeriesGasLimit = "gas_limit" SeriesGasEfficiency = "gas_efficiency" + SeriesNsPfbCount = "pfb_count" + SeriesNsSize = "size" ) //go:generate mockgen -source=$GOFILE -destination=mock/$GOFILE -package=mock -typed @@ -137,4 +139,5 @@ type IStats interface { TPS(ctx context.Context) (TPS, error) TxCountForLast24h(ctx context.Context) ([]TxCountForLast24hItem, error) Series(ctx context.Context, timeframe Timeframe, name string, req SeriesRequest) ([]SeriesItem, error) + NamespaceSeries(ctx context.Context, timeframe Timeframe, name string, nsId uint64, req SeriesRequest) (response []SeriesItem, err error) } diff --git a/internal/storage/types/event_type_enum.go b/internal/storage/types/event_type_enum.go index b2d487dc..c0db8abc 100644 --- a/internal/storage/types/event_type_enum.go +++ b/internal/storage/types/event_type_enum.go @@ -1,6 +1,3 @@ -// SPDX-FileCopyrightText: 2023 PK Lab AG -// SPDX-License-Identifier: MIT - // Code generated by go-enum DO NOT EDIT. // Version: 0.5.7 // Revision: bf63e108589bbd2327b13ec2c5da532aad234029 diff --git a/internal/storage/types/module_enum.go b/internal/storage/types/module_enum.go index 99453b75..0b1b638a 100644 --- a/internal/storage/types/module_enum.go +++ b/internal/storage/types/module_enum.go @@ -1,6 +1,3 @@ -// SPDX-FileCopyrightText: 2023 PK Lab AG -// SPDX-License-Identifier: MIT - // Code generated by go-enum DO NOT EDIT. // Version: 0.5.7 // Revision: bf63e108589bbd2327b13ec2c5da532aad234029 diff --git a/internal/storage/types/msg_address_type_enum.go b/internal/storage/types/msg_address_type_enum.go index b14e0c21..f3143f7e 100644 --- a/internal/storage/types/msg_address_type_enum.go +++ b/internal/storage/types/msg_address_type_enum.go @@ -1,6 +1,3 @@ -// SPDX-FileCopyrightText: 2023 PK Lab AG -// SPDX-License-Identifier: MIT - // Code generated by go-enum DO NOT EDIT. // Version: 0.5.7 // Revision: bf63e108589bbd2327b13ec2c5da532aad234029 diff --git a/internal/storage/types/msg_type_enum.go b/internal/storage/types/msg_type_enum.go index d456bb3d..d55602cf 100644 --- a/internal/storage/types/msg_type_enum.go +++ b/internal/storage/types/msg_type_enum.go @@ -1,6 +1,3 @@ -// SPDX-FileCopyrightText: 2023 PK Lab AG -// SPDX-License-Identifier: MIT - // Code generated by go-enum DO NOT EDIT. // Version: 0.5.7 // Revision: bf63e108589bbd2327b13ec2c5da532aad234029 diff --git a/internal/storage/types/status_enum.go b/internal/storage/types/status_enum.go index aa05fba7..5bf0646f 100644 --- a/internal/storage/types/status_enum.go +++ b/internal/storage/types/status_enum.go @@ -1,6 +1,3 @@ -// SPDX-FileCopyrightText: 2023 PK Lab AG -// SPDX-License-Identifier: MIT - // Code generated by go-enum DO NOT EDIT. // Version: 0.5.7 // Revision: bf63e108589bbd2327b13ec2c5da532aad234029 diff --git a/internal/storage/views.go b/internal/storage/views.go index 687b0309..28b36a48 100644 --- a/internal/storage/views.go +++ b/internal/storage/views.go @@ -4,10 +4,15 @@ package storage const ( - ViewBlockStatsByMinute = "block_stats_by_minute" - ViewBlockStatsByHour = "block_stats_by_hour" - ViewBlockStatsByDay = "block_stats_by_day" - ViewBlockStatsByWeek = "block_stats_by_week" - ViewBlockStatsByMonth = "block_stats_by_month" - ViewBlockStatsByYear = "block_stats_by_year" + ViewBlockStatsByMinute = "block_stats_by_minute" + ViewBlockStatsByHour = "block_stats_by_hour" + ViewBlockStatsByDay = "block_stats_by_day" + ViewBlockStatsByWeek = "block_stats_by_week" + ViewBlockStatsByMonth = "block_stats_by_month" + ViewBlockStatsByYear = "block_stats_by_year" + ViewNamespaceStatsByHour = "namespace_stats_by_hour" + ViewNamespaceStatsByDay = "namespace_stats_by_day" + ViewNamespaceStatsByWeek = "namespace_stats_by_week" + ViewNamespaceStatsByMonth = "namespace_stats_by_month" + ViewNamespaceStatsByYear = "namespace_stats_by_year" ) diff --git a/pkg/indexer/genesis/save.go b/pkg/indexer/genesis/save.go index d3b54460..1ae1eef3 100644 --- a/pkg/indexer/genesis/save.go +++ b/pkg/indexer/genesis/save.go @@ -146,6 +146,7 @@ func (module *Module) save(ctx context.Context, data parsedData) error { Time: messages[i].Time, Height: messages[i].Height, TxId: messages[i].TxId, + Size: uint64(messages[i].Namespace[j].Size), }) } } diff --git a/pkg/indexer/storage/message.go b/pkg/indexer/storage/message.go index 6042e85e..0d9c1dd7 100644 --- a/pkg/indexer/storage/message.go +++ b/pkg/indexer/storage/message.go @@ -51,6 +51,7 @@ func saveMessages( Time: messages[i].Time, Height: messages[i].Height, TxId: messages[i].TxId, + Size: uint64(ns.Size), }) } diff --git a/pkg/node/mock/api.go b/pkg/node/mock/api.go index 0091a6ce..da0d6c19 100644 --- a/pkg/node/mock/api.go +++ b/pkg/node/mock/api.go @@ -1,6 +1,3 @@ -// SPDX-FileCopyrightText: 2023 PK Lab AG -// SPDX-License-Identifier: MIT - // Code generated by MockGen. DO NOT EDIT. // Source: api.go // From 74f782eb2108650d73d2eee31058b5a62b5f1aef Mon Sep 17 00:00:00 2001 From: Artem Date: Fri, 24 Nov 2023 19:43:47 +0100 Subject: [PATCH 2/2] Licences --- cmd/api/docs/docs.go | 3 +++ internal/storage/mock/address.go | 3 +++ internal/storage/mock/balance.go | 3 +++ internal/storage/mock/block.go | 3 +++ internal/storage/mock/block_stats.go | 3 +++ internal/storage/mock/constant.go | 3 +++ internal/storage/mock/denom_metadata.go | 3 +++ internal/storage/mock/event.go | 3 +++ internal/storage/mock/generic.go | 3 +++ internal/storage/mock/message.go | 3 +++ internal/storage/mock/namespace.go | 3 +++ internal/storage/mock/state.go | 3 +++ internal/storage/mock/stats.go | 3 +++ internal/storage/mock/tx.go | 3 +++ internal/storage/mock/validator.go | 3 +++ internal/storage/types/event_type_enum.go | 3 +++ internal/storage/types/module_enum.go | 3 +++ internal/storage/types/msg_address_type_enum.go | 3 +++ internal/storage/types/msg_type_enum.go | 3 +++ internal/storage/types/status_enum.go | 3 +++ pkg/node/mock/api.go | 3 +++ 21 files changed, 63 insertions(+) diff --git a/cmd/api/docs/docs.go b/cmd/api/docs/docs.go index 17cc1b72..a5914ba3 100644 --- a/cmd/api/docs/docs.go +++ b/cmd/api/docs/docs.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2023 PK Lab AG +// SPDX-License-Identifier: MIT + // Package docs Code generated by swaggo/swag. DO NOT EDIT package docs diff --git a/internal/storage/mock/address.go b/internal/storage/mock/address.go index 7d380590..8a6f7ff5 100644 --- a/internal/storage/mock/address.go +++ b/internal/storage/mock/address.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2023 PK Lab AG +// SPDX-License-Identifier: MIT + // Code generated by MockGen. DO NOT EDIT. // Source: address.go // diff --git a/internal/storage/mock/balance.go b/internal/storage/mock/balance.go index 684c1bc2..28bd1ceb 100644 --- a/internal/storage/mock/balance.go +++ b/internal/storage/mock/balance.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2023 PK Lab AG +// SPDX-License-Identifier: MIT + // Code generated by MockGen. DO NOT EDIT. // Source: balance.go // diff --git a/internal/storage/mock/block.go b/internal/storage/mock/block.go index b3af1472..24f7f489 100644 --- a/internal/storage/mock/block.go +++ b/internal/storage/mock/block.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2023 PK Lab AG +// SPDX-License-Identifier: MIT + // Code generated by MockGen. DO NOT EDIT. // Source: block.go // diff --git a/internal/storage/mock/block_stats.go b/internal/storage/mock/block_stats.go index bbf39bbd..acbaf15c 100644 --- a/internal/storage/mock/block_stats.go +++ b/internal/storage/mock/block_stats.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2023 PK Lab AG +// SPDX-License-Identifier: MIT + // Code generated by MockGen. DO NOT EDIT. // Source: block_stats.go // diff --git a/internal/storage/mock/constant.go b/internal/storage/mock/constant.go index ee1891af..0149aa61 100644 --- a/internal/storage/mock/constant.go +++ b/internal/storage/mock/constant.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2023 PK Lab AG +// SPDX-License-Identifier: MIT + // Code generated by MockGen. DO NOT EDIT. // Source: constant.go // diff --git a/internal/storage/mock/denom_metadata.go b/internal/storage/mock/denom_metadata.go index 1c9c6e60..2f2f30ae 100644 --- a/internal/storage/mock/denom_metadata.go +++ b/internal/storage/mock/denom_metadata.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2023 PK Lab AG +// SPDX-License-Identifier: MIT + // Code generated by MockGen. DO NOT EDIT. // Source: denom_metadata.go // diff --git a/internal/storage/mock/event.go b/internal/storage/mock/event.go index 52cdd576..97301152 100644 --- a/internal/storage/mock/event.go +++ b/internal/storage/mock/event.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2023 PK Lab AG +// SPDX-License-Identifier: MIT + // Code generated by MockGen. DO NOT EDIT. // Source: event.go // diff --git a/internal/storage/mock/generic.go b/internal/storage/mock/generic.go index 505c9d6e..7f6dd32c 100644 --- a/internal/storage/mock/generic.go +++ b/internal/storage/mock/generic.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2023 PK Lab AG +// SPDX-License-Identifier: MIT + // Code generated by MockGen. DO NOT EDIT. // Source: generic.go // diff --git a/internal/storage/mock/message.go b/internal/storage/mock/message.go index 02d820a6..4681cd51 100644 --- a/internal/storage/mock/message.go +++ b/internal/storage/mock/message.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2023 PK Lab AG +// SPDX-License-Identifier: MIT + // Code generated by MockGen. DO NOT EDIT. // Source: message.go // diff --git a/internal/storage/mock/namespace.go b/internal/storage/mock/namespace.go index 6787b869..6679865e 100644 --- a/internal/storage/mock/namespace.go +++ b/internal/storage/mock/namespace.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2023 PK Lab AG +// SPDX-License-Identifier: MIT + // Code generated by MockGen. DO NOT EDIT. // Source: namespace.go // diff --git a/internal/storage/mock/state.go b/internal/storage/mock/state.go index 37c546da..5b7e4fe9 100644 --- a/internal/storage/mock/state.go +++ b/internal/storage/mock/state.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2023 PK Lab AG +// SPDX-License-Identifier: MIT + // Code generated by MockGen. DO NOT EDIT. // Source: state.go // diff --git a/internal/storage/mock/stats.go b/internal/storage/mock/stats.go index 014b301e..1fbc94b1 100644 --- a/internal/storage/mock/stats.go +++ b/internal/storage/mock/stats.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2023 PK Lab AG +// SPDX-License-Identifier: MIT + // Code generated by MockGen. DO NOT EDIT. // Source: stats.go // diff --git a/internal/storage/mock/tx.go b/internal/storage/mock/tx.go index 88c02e45..8b9c298a 100644 --- a/internal/storage/mock/tx.go +++ b/internal/storage/mock/tx.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2023 PK Lab AG +// SPDX-License-Identifier: MIT + // Code generated by MockGen. DO NOT EDIT. // Source: tx.go // diff --git a/internal/storage/mock/validator.go b/internal/storage/mock/validator.go index ceb55561..666d5a1c 100644 --- a/internal/storage/mock/validator.go +++ b/internal/storage/mock/validator.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2023 PK Lab AG +// SPDX-License-Identifier: MIT + // Code generated by MockGen. DO NOT EDIT. // Source: validator.go // diff --git a/internal/storage/types/event_type_enum.go b/internal/storage/types/event_type_enum.go index c0db8abc..b2d487dc 100644 --- a/internal/storage/types/event_type_enum.go +++ b/internal/storage/types/event_type_enum.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2023 PK Lab AG +// SPDX-License-Identifier: MIT + // Code generated by go-enum DO NOT EDIT. // Version: 0.5.7 // Revision: bf63e108589bbd2327b13ec2c5da532aad234029 diff --git a/internal/storage/types/module_enum.go b/internal/storage/types/module_enum.go index 0b1b638a..99453b75 100644 --- a/internal/storage/types/module_enum.go +++ b/internal/storage/types/module_enum.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2023 PK Lab AG +// SPDX-License-Identifier: MIT + // Code generated by go-enum DO NOT EDIT. // Version: 0.5.7 // Revision: bf63e108589bbd2327b13ec2c5da532aad234029 diff --git a/internal/storage/types/msg_address_type_enum.go b/internal/storage/types/msg_address_type_enum.go index f3143f7e..b14e0c21 100644 --- a/internal/storage/types/msg_address_type_enum.go +++ b/internal/storage/types/msg_address_type_enum.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2023 PK Lab AG +// SPDX-License-Identifier: MIT + // Code generated by go-enum DO NOT EDIT. // Version: 0.5.7 // Revision: bf63e108589bbd2327b13ec2c5da532aad234029 diff --git a/internal/storage/types/msg_type_enum.go b/internal/storage/types/msg_type_enum.go index d55602cf..d456bb3d 100644 --- a/internal/storage/types/msg_type_enum.go +++ b/internal/storage/types/msg_type_enum.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2023 PK Lab AG +// SPDX-License-Identifier: MIT + // Code generated by go-enum DO NOT EDIT. // Version: 0.5.7 // Revision: bf63e108589bbd2327b13ec2c5da532aad234029 diff --git a/internal/storage/types/status_enum.go b/internal/storage/types/status_enum.go index 5bf0646f..aa05fba7 100644 --- a/internal/storage/types/status_enum.go +++ b/internal/storage/types/status_enum.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2023 PK Lab AG +// SPDX-License-Identifier: MIT + // Code generated by go-enum DO NOT EDIT. // Version: 0.5.7 // Revision: bf63e108589bbd2327b13ec2c5da532aad234029 diff --git a/pkg/node/mock/api.go b/pkg/node/mock/api.go index da0d6c19..0091a6ce 100644 --- a/pkg/node/mock/api.go +++ b/pkg/node/mock/api.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2023 PK Lab AG +// SPDX-License-Identifier: MIT + // Code generated by MockGen. DO NOT EDIT. // Source: api.go //