From 53ca38ce530861e501965d011828843cf4b0c1db Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 5 Sep 2024 11:18:00 +0200 Subject: [PATCH 1/6] Add debug output to duration query --- internal/repository/job.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/internal/repository/job.go b/internal/repository/job.go index e5e25692..1cb4c62e 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -484,10 +484,14 @@ func (r *JobRepository) FindRunningJobs(cluster string) ([]*schema.Job, error) { } func (r *JobRepository) UpdateDuration() error { - if _, err := sq.Update("job"). + stmnt := sq.Update("job"). Set("duration", sq.Expr("? - job.start_time", time.Now().Unix())). - Where("job_state = running"). - RunWith(r.stmtCache).Exec(); err != nil { + Where("job_state = running") + sql, _, err := stmnt.ToSql() + log.Infof("Duration Update query %s", sql) + + _, err = stmnt.RunWith(r.stmtCache).Exec() + if err != nil { return err } From 5e65e21f0bb9387523f46887de607319e0df831a Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 5 Sep 2024 12:38:39 +0200 Subject: [PATCH 2/6] Add quotes in duration query --- internal/repository/job.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/repository/job.go b/internal/repository/job.go index 1cb4c62e..c411ab20 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -486,7 +486,7 @@ func (r *JobRepository) FindRunningJobs(cluster string) ([]*schema.Job, error) { func (r *JobRepository) UpdateDuration() error { stmnt := sq.Update("job"). Set("duration", sq.Expr("? - job.start_time", time.Now().Unix())). - Where("job_state = running") + Where("job_state = 'running'") sql, _, err := stmnt.ToSql() log.Infof("Duration Update query %s", sql) From 7c33dcf630f13c376f56ec5f72f83db9887c5274 Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Thu, 5 Sep 2024 14:58:08 +0200 Subject: [PATCH 3/6] Bugfix in footprint update --- internal/repository/job.go | 4 +--- internal/taskManager/updateFootprintService.go | 7 +++++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/internal/repository/job.go b/internal/repository/job.go index c411ab20..9bad866d 100644 --- a/internal/repository/job.go +++ b/internal/repository/job.go @@ -487,10 +487,8 @@ func (r *JobRepository) UpdateDuration() error { stmnt := sq.Update("job"). Set("duration", sq.Expr("? - job.start_time", time.Now().Unix())). Where("job_state = 'running'") - sql, _, err := stmnt.ToSql() - log.Infof("Duration Update query %s", sql) - _, err = stmnt.RunWith(r.stmtCache).Exec() + _, err := stmnt.RunWith(r.stmtCache).Exec() if err != nil { return err } diff --git a/internal/taskManager/updateFootprintService.go b/internal/taskManager/updateFootprintService.go index 2fdd6b91..2434fd19 100644 --- a/internal/taskManager/updateFootprintService.go +++ b/internal/taskManager/updateFootprintService.go @@ -85,11 +85,14 @@ func RegisterFootprintWorker() { } stmt := sq.Update("job").Where("job.id = ?", job.ID) - if stmt, err = jobRepo.UpdateFootprint(stmt, jobMeta); err != nil { + stmt, err = jobRepo.UpdateFootprint(stmt, jobMeta) + if err != nil { log.Errorf("Update job (dbid: %d) failed at update Footprint step: %s", job.ID, err.Error()) continue } - if stmt, err = jobRepo.UpdateEnergy(stmt, jobMeta); err != nil { + + stmt, err = jobRepo.UpdateEnergy(stmt, jobMeta) + if err != nil { log.Errorf("Update job (dbid: %d) failed at update Energy step: %s", job.ID, err.Error()) continue } From 7ea4086807cc13b08422e303e9d962fdc5fec2ff Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Thu, 5 Sep 2024 15:06:38 +0200 Subject: [PATCH 4/6] Rework sqlite indices in v8 migration --- .../sqlite3/08_add-footprint.up.sql | 54 +++++++++++++++++-- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/internal/repository/migrations/sqlite3/08_add-footprint.up.sql b/internal/repository/migrations/sqlite3/08_add-footprint.up.sql index 48952580..7f0d578a 100644 --- a/internal/repository/migrations/sqlite3/08_add-footprint.up.sql +++ b/internal/repository/migrations/sqlite3/08_add-footprint.up.sql @@ -1,7 +1,11 @@ -CREATE INDEX IF NOT EXISTS jobs_cluster_orderby_starttime ON job (cluster, start_time DESC); -CREATE INDEX IF NOT EXISTS jobs_cluster_count ON job (cluster, job_state, start_time); -CREATE INDEX IF NOT EXISTS jobs_project_orderby_starttime ON job (project, start_time DESC); -CREATE INDEX IF NOT EXISTS jobs_project_count ON job (project, job_state, start_time); +DROP INDEX job_stats; +DROP INDEX job_by_user; +DROP INDEX job_by_starttime; +DROP INDEX job_by_job_id; +DROP INDEX job_list; +DROP INDEX job_list_user; +DROP INDEX job_list_users; +DROP INDEX job_list_users_start; ALTER TABLE job ADD COLUMN energy REAL NOT NULL DEFAULT 0.0; ALTER TABLE job ADD COLUMN energy_footprint TEXT DEFAULT NULL; @@ -27,4 +31,44 @@ ALTER TABLE job DROP net_data_vol_total; ALTER TABLE job DROP file_bw_avg; ALTER TABLE job DROP file_data_vol_total; -PRAGMA optimize; \ No newline at end of file +CREATE INDEX IF NOT EXISTS jobs_cluster ON job (cluster); +CREATE INDEX IF NOT EXISTS jobs_cluster_starttime ON job (cluster, start_time); +CREATE INDEX IF NOT EXISTS jobs_cluster_user ON job (cluster, user); +CREATE INDEX IF NOT EXISTS jobs_cluster_project ON job (cluster, project); +CREATE INDEX IF NOT EXISTS jobs_cluster_subcluster ON job (cluster, subcluster); + +CREATE INDEX IF NOT EXISTS jobs_cluster_partition ON job (cluster, partition); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_starttime ON job (cluster, partition, start_time); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate ON job (cluster, partition, job_state); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_user ON job (cluster, partition, job_state, user); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_project ON job (cluster, partition, job_state, project); +CREATE INDEX IF NOT EXISTS jobs_cluster_partition_jobstate_starttime ON job (cluster, partition, job_state, start_time); + +CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate ON job (cluster, job_state); +CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_starttime ON job (cluster, job_state, start_time); +CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_user ON job (cluster, job_state, user); +CREATE INDEX IF NOT EXISTS jobs_cluster_jobstate_project ON job (cluster, job_state, project); + +CREATE INDEX IF NOT EXISTS jobs_user ON job (user); +CREATE INDEX IF NOT EXISTS jobs_user_starttime ON job (user, start_time); + +CREATE INDEX IF NOT EXISTS jobs_project ON job (project); +CREATE INDEX IF NOT EXISTS jobs_project_starttime ON job (project, start_time); +CREATE INDEX IF NOT EXISTS jobs_project_user ON job (project, user); + +CREATE INDEX IF NOT EXISTS jobs_jobstate ON job (job_state); +CREATE INDEX IF NOT EXISTS jobs_jobstate_user ON job (job_state, user); +CREATE INDEX IF NOT EXISTS jobs_jobstate_project ON job (job_state, project); +CREATE INDEX IF NOT EXISTS jobs_jobstate_cluster ON job (job_state, cluster); +CREATE INDEX IF NOT EXISTS jobs_jobstate_starttime ON job (job_state, start_time); + +CREATE INDEX IF NOT EXISTS jobs_arrayjobid_starttime ON job (array_job_id, start_time); +CREATE INDEX IF NOT EXISTS jobs_cluster_arrayjobid_starttime ON job (cluster, array_job_id, start_time); + +CREATE INDEX IF NOT EXISTS jobs_starttime ON job (start_time); +CREATE INDEX IF NOT EXISTS jobs_duration ON job (duration); +CREATE INDEX IF NOT EXISTS jobs_numnodes ON job (num_nodes); +CREATE INDEX IF NOT EXISTS jobs_numhwthreads ON job (num_hwthreads); +CREATE INDEX IF NOT EXISTS jobs_numacc ON job (num_acc); + +PRAGMA optimize; From 5482b9be2c603571bea15fa49856b0d75a3bc2de Mon Sep 17 00:00:00 2001 From: Jan Eitzinger Date: Fri, 6 Sep 2024 11:24:54 +0200 Subject: [PATCH 5/6] Add debug output --- internal/taskManager/updateFootprintService.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/taskManager/updateFootprintService.go b/internal/taskManager/updateFootprintService.go index 2434fd19..55f4cef0 100644 --- a/internal/taskManager/updateFootprintService.go +++ b/internal/taskManager/updateFootprintService.go @@ -47,9 +47,10 @@ func RegisterFootprintWorker() { scopes = append(scopes, schema.MetricScopeAccelerator) for _, job := range jobs { + log.Debugf("Try job %d", job.JobID) jobData, err := metricDataDispatcher.LoadData(job, allMetrics, scopes, context.Background()) if err != nil { - log.Error("Error wile loading job data for footprint update") + log.Errorf("Error wile loading job data for footprint update: %v", err) continue } @@ -107,6 +108,7 @@ func RegisterFootprintWorker() { // log.Errorf("Update job (dbid: %d) failed at db execute: %s", job.ID, err.Error()) // continue // } + log.Debugf("Finish job %d", job.JobID) } jobRepo.TransactionCommit(t) From 1b5f4bff2c092e4910ef0dbd72b8f13f9bb73f66 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 9 Sep 2024 10:32:26 +0200 Subject: [PATCH 6/6] feat: SyncUserOnLogin now updates name of token logged user --- internal/auth/auth.go | 8 ++++++-- internal/repository/user.go | 21 +++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/internal/auth/auth.go b/internal/auth/auth.go index e45fa9df..b6e4cbe6 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -145,14 +145,18 @@ func GetAuthInstance() *Authentication { func persistUser(user *schema.User) { r := repository.GetUserRepository() - _, err := r.GetUser(user.Username) + dbUser, err := r.GetUser(user.Username) if err != nil && err != sql.ErrNoRows { log.Errorf("Error while loading user '%s': %v", user.Username, err) - } else if err == sql.ErrNoRows { + } else if err == sql.ErrNoRows { // Adds New User if err := r.AddUser(user); err != nil { log.Errorf("Error while adding user '%s' to DB: %v", user.Username, err) } + } else { // Update Existing + if err := r.UpdateUser(dbUser, user); err != nil { + log.Errorf("Error while updating user '%s' to DB: %v", user.Username, err) + } } } diff --git a/internal/repository/user.go b/internal/repository/user.go index a8776a92..db961368 100644 --- a/internal/repository/user.go +++ b/internal/repository/user.go @@ -130,6 +130,27 @@ func (r *UserRepository) AddUser(user *schema.User) error { return nil } +func (r *UserRepository) UpdateUser(dbUser *schema.User, user *schema.User) error { + // user contains updated info, apply to dbuser + // TODO: Discuss updatable fields + if dbUser.Name != user.Name { + if _, err := sq.Update("user").Set("name", user.Name).Where("user.username = ?", dbUser.Username).RunWith(r.DB).Exec(); err != nil { + log.Errorf("error while updating name of user '%s'", user.Username) + return err + } + } + + // Toggled until greenlit + // if dbUser.HasRole(schema.RoleManager) && !reflect.DeepEqual(dbUser.Projects, user.Projects) { + // projects, _ := json.Marshal(user.Projects) + // if _, err := sq.Update("user").Set("projects", projects).Where("user.username = ?", dbUser.Username).RunWith(r.DB).Exec(); err != nil { + // return err + // } + // } + + return nil +} + func (r *UserRepository) DelUser(username string) error { _, err := r.DB.Exec(`DELETE FROM user WHERE user.username = ?`, username) if err != nil {