From 8f6b2500309e3257da6abdcdfb949c343f1b8433 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thilo=20Schmalfu=C3=9F?= Date: Wed, 15 Nov 2023 17:18:34 +0100 Subject: [PATCH] [sessions] Session cleanup (#604) * Share Intellij run configurations * [sessions] Cleanup inactive sessions after 168h * Create a TTL index on the updatedAt field * Change type of createdAt and updatedAt to Date (required for TTL index) * Regenerated mocks => the session cleanup will only work for sessions create/touch after this change. => older sessions need to dropped manually see: https://www.mongodb.com/docs/manual/core/index-ttl/ --- .run/cluster.run.xml | 12 +++++ .run/hub.run.xml | 12 +++++ .run/start all.run.xml | 8 ++++ .run/watcher.run.xml | 12 +++++ CONTRIBUTING.md | 6 ++- cmd/kobs/hub/hub.go | 8 +++- pkg/hub/db/db.go | 22 +++++++++ pkg/hub/db/db_mock.go | 14 ++++++ pkg/hub/db/db_test.go | 9 ++++ pkg/hub/db/sessions.go | 15 +++--- .../prometheus/instance/instance_mock.go | 47 +++++-------------- .../sonarqube/instance/instance_mock.go | 8 ++-- 12 files changed, 126 insertions(+), 47 deletions(-) create mode 100644 .run/cluster.run.xml create mode 100644 .run/hub.run.xml create mode 100644 .run/start all.run.xml create mode 100644 .run/watcher.run.xml diff --git a/.run/cluster.run.xml b/.run/cluster.run.xml new file mode 100644 index 000000000..138de5baa --- /dev/null +++ b/.run/cluster.run.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/.run/hub.run.xml b/.run/hub.run.xml new file mode 100644 index 000000000..2c64985e0 --- /dev/null +++ b/.run/hub.run.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/.run/start all.run.xml b/.run/start all.run.xml new file mode 100644 index 000000000..fe90a9ef5 --- /dev/null +++ b/.run/start all.run.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.run/watcher.run.xml b/.run/watcher.run.xml new file mode 100644 index 000000000..2b36d2b8f --- /dev/null +++ b/.run/watcher.run.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7e1a751ce..15d9b528a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -165,7 +165,11 @@ The frontend lives in the [`plugins/app`](./plugins/app) folder. The shared Reac - `yarn workspace @kobsio/app build`: Build the React UI. - `yarn workspace @kobsio/app start`: Start development server for the React UI. The development server is served on port `3000`. -We are using [ESLint](https://eslint.org) and [Prettier](https://prettier.io) for linting and automatic code formation. When you are using [VS Code](https://code.visualstudio.com) you can also use the `launch.json` file from the `.vscode` folder for debugging the React UI. +We are using [ESLint](https://eslint.org) and [Prettier](https://prettier.io) for linting and automatic code formation. + +When you are using [VS Code](https://code.visualstudio.com) you can also use the `launch.json` file from the `.vscode` folder for debugging the React UI. + +For [Intellj](https://www.jetbrains.com/idea/) users the `.run` folder contains shared run configurations. #### Plugins diff --git a/cmd/kobs/hub/hub.go b/cmd/kobs/hub/hub.go index b1e957dd6..d85cdb612 100644 --- a/cmd/kobs/hub/hub.go +++ b/cmd/kobs/hub/hub.go @@ -83,6 +83,12 @@ func (r *Cmd) Run(plugins []plugins.Plugin) error { return err } + err = dbClient.CreateIndexes(context.Background()) + if err != nil { + log.Error(context.Background(), "Could not create indexes", zap.Error(err)) + return err + } + pluginsClient, err := hubPlugins.NewClient(plugins, cfg.Hub.Plugins, clustersClient, dbClient) if err != nil { log.Error(context.Background(), "Could not create plugins client", zap.Error(err)) @@ -115,7 +121,7 @@ func (r *Cmd) Run(plugins []plugins.Plugin) error { done := make(chan os.Signal, 1) signal.Notify(done, os.Interrupt, syscall.SIGTERM) - log.Debug(context.Background(), "Start listining for SIGINT and SIGTERM signal") + log.Debug(context.Background(), "Start listening for SIGINT and SIGTERM signal") <-done log.Info(context.Background(), "Shutdown kobs hub...") diff --git a/pkg/hub/db/db.go b/pkg/hub/db/db.go index 393e90602..35e3ad30c 100644 --- a/pkg/hub/db/db.go +++ b/pkg/hub/db/db.go @@ -34,6 +34,7 @@ type Config struct { // Client is the interface with all the methods to interact with the db. type Client interface { DB() *mongo.Client + CreateIndexes(ctx context.Context) error SavePlugins(ctx context.Context, cluster string, plugins []plugin.Instance) error SaveNamespaces(ctx context.Context, cluster string, namespaces []string) error SaveCRDs(ctx context.Context, crds []kubernetes.CRD) error @@ -126,6 +127,27 @@ func (c *client) DB() *mongo.Client { return c.db } +func (c *client) CreateIndexes(ctx context.Context) error { + ctx, span := c.tracer.Start(ctx, "db.CreateIndexes") + defer span.End() + + // Create TTL index for sessions collection, which will delete all inactive sessions which are older than 7 days (168h). + _, err := c.db.Database("kobs").Collection("sessions").Indexes().CreateOne( + ctx, + mongo.IndexModel{ + Keys: bson.D{{Key: "updatedAt", Value: 1}}, + Options: options.Index().SetExpireAfterSeconds(int32(time.Duration(168 * time.Hour).Seconds())), + }) + + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + return err + } + + return nil +} + func (c *client) SavePlugins(ctx context.Context, cluster string, plugins []plugin.Instance) error { if len(plugins) == 0 { return nil diff --git a/pkg/hub/db/db_mock.go b/pkg/hub/db/db_mock.go index ea448db00..a377d7669 100644 --- a/pkg/hub/db/db_mock.go +++ b/pkg/hub/db/db_mock.go @@ -43,6 +43,20 @@ func (m *MockClient) EXPECT() *MockClientMockRecorder { return m.recorder } +// CreateIndexes mocks base method. +func (m *MockClient) CreateIndexes(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateIndexes", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateIndexes indicates an expected call of CreateIndexes. +func (mr *MockClientMockRecorder) CreateIndexes(ctx interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateIndexes", reflect.TypeOf((*MockClient)(nil).CreateIndexes), ctx) +} + // CreateSession mocks base method. func (m *MockClient) CreateSession(ctx context.Context, user context0.User) (*Session, error) { m.ctrl.T.Helper() diff --git a/pkg/hub/db/db_test.go b/pkg/hub/db/db_test.go index 2c91b9943..6171e5cf9 100644 --- a/pkg/hub/db/db_test.go +++ b/pkg/hub/db/db_test.go @@ -46,6 +46,15 @@ func TestNewClient(t *testing.T) { require.NotEmpty(t, c2) } +func TestCreateIndexes(t *testing.T) { + uri, container := setupDatabase(t) + defer gnomock.Stop(container) + c, _ := NewClient(Config{URI: uri}) + + err := c.CreateIndexes(context.Background()) + require.NoError(t, err) +} + func TestSaveAndGetPlugins(t *testing.T) { plugins := []plugin.Instance{{ Name: "test-cluster", diff --git a/pkg/hub/db/sessions.go b/pkg/hub/db/sessions.go index fbb1ad114..dc8c0ce4a 100644 --- a/pkg/hub/db/sessions.go +++ b/pkg/hub/db/sessions.go @@ -15,28 +15,29 @@ import ( ) var ( - // ErrSessionNotFound is our custom error which is returned when we are not able to found a session with the + // ErrSessionNotFound is our custom error which is returned when we are not able to find a session with the // provided session id. ErrSessionNotFound = fmt.Errorf("session not found") ) -// Session is the structure of a single session as it is saved in the database. Each session contains an id and the a +// Session is the structure of a single session as it is saved in the database. Each session contains an id and a // user to which the session belongs to. The session also contains a `createdAt` and `updatedAt` field, so that we know // when a session was created or used the last time. type Session struct { ID primitive.ObjectID `json:"id" bson:"_id"` User authContext.User `json:"user" bson:"user"` - CreatedAt int64 `json:"createdAt" bson:"createdAt"` - UpdatedAt int64 `json:"updatedAt" bson:"updatedAt"` + CreatedAt time.Time `json:"createdAt" bson:"createdAt"` + UpdatedAt time.Time `json:"updatedAt" bson:"updatedAt"` } // CreateSession creates a new session for the provided `user`. func (c *client) CreateSession(ctx context.Context, user authContext.User) (*Session, error) { + now := time.Now() session := Session{ ID: primitive.NewObjectID(), User: user, - CreatedAt: time.Now().Unix(), - UpdatedAt: time.Now().Unix(), + CreatedAt: now, + UpdatedAt: now, } ctx, span := c.tracer.Start(ctx, "db.CreateSession") @@ -86,7 +87,7 @@ func (c *client) GetAndUpdateSession(ctx context.Context, sessionID primitive.Ob span.SetAttributes(attribute.Key("sessionID").String(sessionID.String())) defer span.End() - res := c.db.Database("kobs").Collection("sessions").FindOneAndUpdate(ctx, bson.D{{Key: "_id", Value: sessionID}}, bson.D{{Key: "$set", Value: bson.D{{Key: "updatedAt", Value: time.Now().Unix()}}}}) + res := c.db.Database("kobs").Collection("sessions").FindOneAndUpdate(ctx, bson.D{{Key: "_id", Value: sessionID}}, bson.D{{Key: "$set", Value: bson.D{{Key: "updatedAt", Value: time.Now()}}}}) if res.Err() != nil { span.RecordError(res.Err()) span.SetStatus(codes.Error, res.Err().Error()) diff --git a/pkg/plugins/prometheus/instance/instance_mock.go b/pkg/plugins/prometheus/instance/instance_mock.go index 53c2fdcbb..92532b116 100644 --- a/pkg/plugins/prometheus/instance/instance_mock.go +++ b/pkg/plugins/prometheus/instance/instance_mock.go @@ -6,11 +6,10 @@ package instance import ( context "context" + http "net/http" reflect "reflect" gomock "github.com/golang/mock/gomock" - v1 "github.com/prometheus/client_golang/api/prometheus/v1" - model "github.com/prometheus/common/model" ) // MockInstance is a mock of Instance interface. @@ -51,22 +50,6 @@ func (mr *MockInstanceMockRecorder) GetInstant(ctx, queries, timeEnd interface{} return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInstant", reflect.TypeOf((*MockInstance)(nil).GetInstant), ctx, queries, timeEnd) } -// GetLabels mocks base method. -func (m *MockInstance) GetLabels(ctx context.Context, matches []string, timeStart, timeEnd int64) ([]string, v1.Warnings, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetLabels", ctx, matches, timeStart, timeEnd) - ret0, _ := ret[0].([]string) - ret1, _ := ret[1].(v1.Warnings) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// GetLabels indicates an expected call of GetLabels. -func (mr *MockInstanceMockRecorder) GetLabels(ctx, matches, timeStart, timeEnd interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLabels", reflect.TypeOf((*MockInstance)(nil).GetLabels), ctx, matches, timeStart, timeEnd) -} - // GetName mocks base method. func (m *MockInstance) GetName() string { m.ctrl.T.Helper() @@ -96,22 +79,6 @@ func (mr *MockInstanceMockRecorder) GetRange(ctx, queries, resolution, timeStart return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRange", reflect.TypeOf((*MockInstance)(nil).GetRange), ctx, queries, resolution, timeStart, timeEnd) } -// GetSeries mocks base method. -func (m *MockInstance) GetSeries(ctx context.Context, matches []string, timeStart, timeEnd int64) ([]model.LabelSet, v1.Warnings, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetSeries", ctx, matches, timeStart, timeEnd) - ret0, _ := ret[0].([]model.LabelSet) - ret1, _ := ret[1].(v1.Warnings) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// GetSeries indicates an expected call of GetSeries. -func (mr *MockInstanceMockRecorder) GetSeries(ctx, matches, timeStart, timeEnd interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSeries", reflect.TypeOf((*MockInstance)(nil).GetSeries), ctx, matches, timeStart, timeEnd) -} - // GetVariable mocks base method. func (m *MockInstance) GetVariable(ctx context.Context, label, query, queryType string, timeStart, timeEnd int64) ([]string, error) { m.ctrl.T.Helper() @@ -126,3 +93,15 @@ func (mr *MockInstanceMockRecorder) GetVariable(ctx, label, query, queryType, ti mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVariable", reflect.TypeOf((*MockInstance)(nil).GetVariable), ctx, label, query, queryType, timeStart, timeEnd) } + +// Proxy mocks base method. +func (m *MockInstance) Proxy(w http.ResponseWriter, r *http.Request) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Proxy", w, r) +} + +// Proxy indicates an expected call of Proxy. +func (mr *MockInstanceMockRecorder) Proxy(w, r interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Proxy", reflect.TypeOf((*MockInstance)(nil).Proxy), w, r) +} diff --git a/pkg/plugins/sonarqube/instance/instance_mock.go b/pkg/plugins/sonarqube/instance/instance_mock.go index 0ea508a60..35814d6d8 100644 --- a/pkg/plugins/sonarqube/instance/instance_mock.go +++ b/pkg/plugins/sonarqube/instance/instance_mock.go @@ -64,16 +64,16 @@ func (mr *MockInstanceMockRecorder) GetProjectMeasures(ctx, project, metricKeys } // GetProjects mocks base method. -func (m *MockInstance) GetProjects(ctx context.Context, query, pageSize, pageNumber string) (*ResponseProjects, error) { +func (m *MockInstance) GetProjects(ctx context.Context, query, page, perPage string) (*ResponseProjects, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetProjects", ctx, query, pageSize, pageNumber) + ret := m.ctrl.Call(m, "GetProjects", ctx, query, page, perPage) ret0, _ := ret[0].(*ResponseProjects) ret1, _ := ret[1].(error) return ret0, ret1 } // GetProjects indicates an expected call of GetProjects. -func (mr *MockInstanceMockRecorder) GetProjects(ctx, query, pageSize, pageNumber interface{}) *gomock.Call { +func (mr *MockInstanceMockRecorder) GetProjects(ctx, query, page, perPage interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProjects", reflect.TypeOf((*MockInstance)(nil).GetProjects), ctx, query, pageSize, pageNumber) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProjects", reflect.TypeOf((*MockInstance)(nil).GetProjects), ctx, query, page, perPage) }