-
Notifications
You must be signed in to change notification settings - Fork 14
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'dev' into sample_resolution_select
- Moved resample changes to metricDataDispatcher - Added res argument to archiver, updateFootprintService
- Loading branch information
Showing
25 changed files
with
824 additions
and
524 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,94 @@ | ||
| // Copyright (C) NHR@FAU, University Erlangen-Nuremberg. | ||
| // All rights reserved. | ||
| // Use of this source code is governed by a MIT-style | ||
| // license that can be found in the LICENSE file. | ||
| package archiver | ||
|
|
||
| import ( | ||
| "context" | ||
| "sync" | ||
| "time" | ||
|
|
||
| "github.com/ClusterCockpit/cc-backend/internal/repository" | ||
| "github.com/ClusterCockpit/cc-backend/pkg/log" | ||
| "github.com/ClusterCockpit/cc-backend/pkg/schema" | ||
| sq "github.com/Masterminds/squirrel" | ||
| ) | ||
|
|
||
| var ( | ||
| archivePending sync.WaitGroup | ||
| archiveChannel chan *schema.Job | ||
| jobRepo *repository.JobRepository | ||
| ) | ||
|
|
||
| func Start(r *repository.JobRepository) { | ||
| archiveChannel = make(chan *schema.Job, 128) | ||
| jobRepo = r | ||
|
|
||
| go archivingWorker() | ||
| } | ||
|
|
||
| // Archiving worker thread | ||
| func archivingWorker() { | ||
| for { | ||
| select { | ||
| case job, ok := <-archiveChannel: | ||
| if !ok { | ||
| break | ||
| } | ||
| start := time.Now() | ||
| // not using meta data, called to load JobMeta into Cache? | ||
| // will fail if job meta not in repository | ||
| if _, err := jobRepo.FetchMetadata(job); err != nil { | ||
| log.Errorf("archiving job (dbid: %d) failed at check metadata step: %s", job.ID, err.Error()) | ||
| jobRepo.UpdateMonitoringStatus(job.ID, schema.MonitoringStatusArchivingFailed) | ||
| continue | ||
| } | ||
|
|
||
| // ArchiveJob will fetch all the data from a MetricDataRepository and push into configured archive backend | ||
| // TODO: Maybe use context with cancel/timeout here | ||
| jobMeta, err := ArchiveJob(job, context.Background()) | ||
| if err != nil { | ||
| log.Errorf("archiving job (dbid: %d) failed at archiving job step: %s", job.ID, err.Error()) | ||
| jobRepo.UpdateMonitoringStatus(job.ID, schema.MonitoringStatusArchivingFailed) | ||
| continue | ||
| } | ||
|
|
||
| stmt := sq.Update("job").Where("job.id = ?", job.ID) | ||
|
|
||
| if stmt, err = jobRepo.UpdateFootprint(stmt, jobMeta); err != nil { | ||
| log.Errorf("archiving job (dbid: %d) failed at update Footprint step: %s", job.ID, err.Error()) | ||
| continue | ||
| } | ||
| if stmt, err = jobRepo.UpdateEnergy(stmt, jobMeta); err != nil { | ||
| log.Errorf("archiving job (dbid: %d) failed at update Energy step: %s", job.ID, err.Error()) | ||
| continue | ||
| } | ||
| // Update the jobs database entry one last time: | ||
| stmt = jobRepo.MarkArchived(stmt, schema.MonitoringStatusArchivingSuccessful) | ||
| if err := jobRepo.Execute(stmt); err != nil { | ||
| log.Errorf("archiving job (dbid: %d) failed at db execute: %s", job.ID, err.Error()) | ||
| continue | ||
| } | ||
| log.Debugf("archiving job %d took %s", job.JobID, time.Since(start)) | ||
| log.Printf("archiving job (dbid: %d) successful", job.ID) | ||
| archivePending.Done() | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // Trigger async archiving | ||
| func TriggerArchiving(job *schema.Job) { | ||
| if archiveChannel == nil { | ||
| log.Fatal("Cannot archive without archiving channel. Did you Start the archiver?") | ||
| } | ||
|
|
||
| archivePending.Add(1) | ||
| archiveChannel <- job | ||
| } | ||
|
|
||
| // Wait for background thread to finish pending archiving operations | ||
| func WaitForArchiving() { | ||
| // close channel and wait for worker to process remaining jobs | ||
| archivePending.Wait() | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,82 @@ | ||
| // Copyright (C) NHR@FAU, University Erlangen-Nuremberg. | ||
| // All rights reserved. | ||
| // Use of this source code is governed by a MIT-style | ||
| // license that can be found in the LICENSE file. | ||
| package archiver | ||
|
|
||
| import ( | ||
| "context" | ||
| "math" | ||
|
|
||
| "github.com/ClusterCockpit/cc-backend/internal/config" | ||
| "github.com/ClusterCockpit/cc-backend/internal/metricDataDispatcher" | ||
| "github.com/ClusterCockpit/cc-backend/pkg/archive" | ||
| "github.com/ClusterCockpit/cc-backend/pkg/log" | ||
| "github.com/ClusterCockpit/cc-backend/pkg/schema" | ||
| ) | ||
|
|
||
| // Writes a running job to the job-archive | ||
| func ArchiveJob(job *schema.Job, ctx context.Context) (*schema.JobMeta, error) { | ||
| allMetrics := make([]string, 0) | ||
| metricConfigs := archive.GetCluster(job.Cluster).MetricConfig | ||
| for _, mc := range metricConfigs { | ||
| allMetrics = append(allMetrics, mc.Name) | ||
| } | ||
|
|
||
| scopes := []schema.MetricScope{schema.MetricScopeNode} | ||
| // FIXME: Add a config option for this | ||
| if job.NumNodes <= 8 { | ||
| // This will add the native scope if core scope is not available | ||
| scopes = append(scopes, schema.MetricScopeCore) | ||
| } | ||
|
|
||
| if job.NumAcc > 0 { | ||
| scopes = append(scopes, schema.MetricScopeAccelerator) | ||
| } | ||
|
|
||
| jobData, err := metricDataDispatcher.LoadData(job, allMetrics, scopes, ctx) | ||
| if err != nil { | ||
| log.Error("Error wile loading job data for archiving") | ||
| return nil, err | ||
| } | ||
|
|
||
| jobMeta := &schema.JobMeta{ | ||
| BaseJob: job.BaseJob, | ||
| StartTime: job.StartTime.Unix(), | ||
| Statistics: make(map[string]schema.JobStatistics), | ||
| } | ||
|
|
||
| for metric, data := range jobData { | ||
| avg, min, max := 0.0, math.MaxFloat32, -math.MaxFloat32 | ||
| nodeData, ok := data["node"] | ||
| if !ok { | ||
| // This should never happen ? | ||
| continue | ||
| } | ||
|
|
||
| for _, series := range nodeData.Series { | ||
| avg += series.Statistics.Avg | ||
| min = math.Min(min, series.Statistics.Min) | ||
| max = math.Max(max, series.Statistics.Max) | ||
| } | ||
|
|
||
| jobMeta.Statistics[metric] = schema.JobStatistics{ | ||
| Unit: schema.Unit{ | ||
| Prefix: archive.GetMetricConfig(job.Cluster, metric).Unit.Prefix, | ||
| Base: archive.GetMetricConfig(job.Cluster, metric).Unit.Base, | ||
| }, | ||
| Avg: avg / float64(job.NumNodes), | ||
| Min: min, | ||
| Max: max, | ||
| } | ||
| } | ||
|
|
||
| // If the file based archive is disabled, | ||
| // only return the JobMeta structure as the | ||
| // statistics in there are needed. | ||
| if config.Keys.DisableArchive { | ||
| return jobMeta, nil | ||
| } | ||
|
|
||
| return jobMeta, archive.GetHandle().ImportJob(jobMeta, &jobData) | ||
| } |
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.