diff --git a/cmd/collectors/rest/plugins/auditlog/auditlog.go b/cmd/collectors/rest/plugins/auditlog/auditlog.go new file mode 100644 index 000000000..5b8e0fbf5 --- /dev/null +++ b/cmd/collectors/rest/plugins/auditlog/auditlog.go @@ -0,0 +1,274 @@ +package auditlog + +import ( + "fmt" + "github.com/netapp/harvest/v2/cmd/collectors" + "github.com/netapp/harvest/v2/cmd/poller/plugin" + "github.com/netapp/harvest/v2/cmd/tools/rest" + "github.com/netapp/harvest/v2/pkg/conf" + "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/slogx" + "github.com/netapp/harvest/v2/pkg/util" + "log/slog" + "maps" + "strings" + "time" +) + +const ( + auditMatrix = "audit" + defaultDataPollDuration = 3 * time.Minute +) + +var defaultFields = []string{"application", "location", "user", "timestamp", "state"} + +type VolumeCache struct { + cache map[string]VolumeInfo + cacheCopy map[string]VolumeInfo + hasCacheRefreshed bool +} + +type VolumeInfo struct { + name string + svm string +} + +type AuditLog struct { + *plugin.AbstractPlugin + schedule int + data *matrix.Matrix + client *rest.Client + rootConfig RootConfig + lastFilterTimes map[string]int64 + volumeCache VolumeCache +} + +var metrics = []string{ + "log", +} + +func New(p *plugin.AbstractPlugin) plugin.Plugin { + return &AuditLog{AbstractPlugin: p} +} + +func (a *AuditLog) Init(remote conf.Remote) error { + if err := a.AbstractPlugin.Init(remote); err != nil { + return err + } + + err := a.initMatrix() + if err != nil { + return err + } + + err = a.populateAuditLogConfig() + if err != nil { + return err + } + + timeout, _ := time.ParseDuration(rest.DefaultTimeout) + if a.client, err = rest.New(conf.ZapiPoller(a.ParentParams), timeout, a.Auth); err != nil { + return err + } + a.schedule = a.SetPluginInterval() + a.lastFilterTimes = make(map[string]int64) + a.InitVolumeCache() + + return a.client.Init(5, remote) +} + +func (a *AuditLog) InitVolumeCache() { + a.volumeCache = VolumeCache{ + cache: make(map[string]VolumeInfo), + cacheCopy: make(map[string]VolumeInfo), + hasCacheRefreshed: false, + } +} + +func (a *AuditLog) initMatrix() error { + a.data = matrix.New(a.Parent+auditMatrix, auditMatrix, auditMatrix) + a.data.SetExportOptions(matrix.DefaultExportOptions()) + for _, k := range metrics { + err := matrix.CreateMetric(k, a.data) + if err != nil { + a.SLogger.Warn("error while creating metric", slogx.Err(err), slog.String("key", k)) + return err + } + } + return nil +} + +func (a *AuditLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { + a.client.Metadata.Reset() + + if a.schedule >= a.PluginInvocationRate { + a.schedule = 0 + err := a.populateVolumeCache() + if err != nil { + return nil, nil, err + } + } + a.schedule++ + + err := a.initMatrix() + if err != nil { + return nil, nil, err + } + data := dataMap[a.Object] + a.data.SetGlobalLabels(data.GetGlobalLabels()) + + clusterTime, err := collectors.GetClusterTime(a.client, nil, a.SLogger) + if err != nil { + return nil, nil, err + } + + if a.HasVolumeConfig() { + a.volumeCache.hasCacheRefreshed = false + // process volume rootConfig + volume := a.rootConfig.AuditLog.Volume + var actions = make([]string, len(volume.Action)) + for _, action := range volume.Action { + actions = append(actions, fmt.Sprintf("*%s*", action)) + } + state := volume.Filter.State + timestampFilter := a.getTimeStampFilter(clusterTime, a.lastFilterTimes["volume"]) + href := a.constructAuditLogURL(actions, state, timestampFilter) + records, err := collectors.InvokeRestCall(a.client, href) + if err != nil { + return nil, nil, err + } + a.parseVolumeRecords(records) + // update lastFilterTime to current cluster time + a.lastFilterTimes["volume"] = clusterTime.Unix() + } + + return []*matrix.Matrix{a.data}, a.client.Metadata, nil +} + +func (a *AuditLog) HasVolumeConfig() bool { + return len(a.rootConfig.AuditLog.Volume.Action) > 0 +} + +func (a *AuditLog) populateAuditLogConfig() error { + var err error + + a.rootConfig, err = InitAuditLogConfig() + if err != nil { + return err + } + return nil +} + +func (a *AuditLog) constructAuditLogURL(actions []string, state string, timestampFilter string) string { + actionFilter := "input=" + strings.Join(actions, "|") + stateFilter := "state=" + state + + // Construct the Href + href := rest.NewHrefBuilder(). + APIPath("api/security/audit/messages"). + Fields(defaultFields). + Filter([]string{timestampFilter, actionFilter, stateFilter}). + Build() + + return href +} + +func (a *AuditLog) getTimeStampFilter(clusterTime time.Time, lastFilterTime int64) string { + fromTime := lastFilterTime + // check if this is the first request + if lastFilterTime == 0 { + // if first request fetch cluster time + dataDuration, err := collectors.GetDataInterval(a.ParentParams, defaultDataPollDuration) + if err != nil { + a.SLogger.Warn( + "Failed to parse duration. using default", + slogx.Err(err), + slog.String("defaultDataPollDuration", defaultDataPollDuration.String()), + ) + } + fromTime = clusterTime.Add(-dataDuration).Unix() + } + return fmt.Sprintf("timestamp=>=%d", fromTime) +} + +func (a *AuditLog) populateVolumeCache() error { + // Initialize cacheCopy to store elements that will be removed from cache + a.volumeCache.cacheCopy = make(map[string]VolumeInfo) + + // Clone the existing cache to compare later + oldCache := maps.Clone(a.volumeCache.cache) + + a.volumeCache.cache = make(map[string]VolumeInfo) + + query := "api/storage/volumes" + href := rest.NewHrefBuilder(). + APIPath(query). + MaxRecords(collectors.DefaultBatchSize). + Fields([]string{"svm.name", "uuid", "name"}). + Build() + + records, err := rest.FetchAll(a.client, href) + if err != nil { + return err + } + + if len(records) == 0 { + return nil + } + + for _, volume := range records { + if !volume.IsObject() { + a.SLogger.Warn("volume is not object, skipping", slog.String("type", volume.Type.String())) + continue + } + uuid := volume.Get("uuid").ClonedString() + name := volume.Get("name").ClonedString() + svm := volume.Get("svm.name").ClonedString() + + // Update the cache with the current volume info + a.volumeCache.cache[uuid] = VolumeInfo{ + name: name, + svm: svm, + } + } + + // Identify elements that were in the old cache but are not in the new cache + for uuid, volumeInfo := range oldCache { + if _, exists := a.volumeCache.cache[uuid]; !exists { + a.volumeCache.cacheCopy[uuid] = volumeInfo + } + } + + return nil +} + +func (a *AuditLog) GetVolumeInfo(uuid string) (VolumeInfo, bool) { + volumeInfo, exists := a.volumeCache.cache[uuid] + if exists { + return volumeInfo, true + } + volumeInfo, exists = a.volumeCache.cacheCopy[uuid] + return volumeInfo, exists +} + +func (a *AuditLog) setLogMetric(mat *matrix.Matrix, instance *matrix.Instance, value float64) error { + m := mat.GetMetric("log") + if m != nil { + if err := m.SetValueFloat64(instance, value); err != nil { + return err + } + } + return nil +} + +func (a *AuditLog) RefreshVolumeCache(refreshCache bool) error { + if refreshCache && !a.volumeCache.hasCacheRefreshed { + a.SLogger.Info("refreshing cache via handler") + err := a.populateVolumeCache() + if err != nil { + return err + } + a.volumeCache.hasCacheRefreshed = true + } + return nil +} diff --git a/cmd/collectors/rest/plugins/auditlog/auditlog_helper.go b/cmd/collectors/rest/plugins/auditlog/auditlog_helper.go new file mode 100644 index 000000000..5e871d047 --- /dev/null +++ b/cmd/collectors/rest/plugins/auditlog/auditlog_helper.go @@ -0,0 +1,56 @@ +package auditlog + +import ( + "gopkg.in/yaml.v3" +) + +type Filter struct { + State string `yaml:"state"` +} + +type Volume struct { + Action []string `yaml:"action"` + Filter Filter `yaml:"filter"` +} + +type Login struct { + Action []string `yaml:"action"` + Filter Filter `yaml:"filter"` +} + +type Config struct { + Volume Volume `yaml:"Volume"` + Login Login `yaml:"Login"` +} + +type RootConfig struct { + AuditLog Config `yaml:"AuditLog"` +} + +func InitAuditLogConfig() (RootConfig, error) { + data := ` +AuditLog: + Volume: + action: + - PATCH /api/storage/volumes + - POST /api/application/applications + - POST /api/storage/volumes + - volume create + - volume modify + - volume delete + - POST /api/private/cli/volume + - DELETE /api/private/cli/volume + - POST /api/private/cli/volume/rename + - DELETE /api/storage/volumes + filter: + state: success +` + + var config RootConfig + err := yaml.Unmarshal([]byte(data), &config) + if err != nil { + return RootConfig{}, err + } + + return config, nil +} diff --git a/cmd/collectors/rest/plugins/auditlog/auditlog_volume.go b/cmd/collectors/rest/plugins/auditlog/auditlog_volume.go new file mode 100644 index 000000000..d0e587040 --- /dev/null +++ b/cmd/collectors/rest/plugins/auditlog/auditlog_volume.go @@ -0,0 +1,446 @@ +package auditlog + +import ( + "encoding/json" + "github.com/netapp/harvest/v2/pkg/slogx" + "github.com/netapp/harvest/v2/third_party/tidwall/gjson" + "log/slog" + "regexp" + "strings" + "time" +) + +var ( + replacer = strings.NewReplacer("^I", " ", `\"`, `"`) +) + +type VolumeHandler interface { + ExtractNames(string, *AuditLog) (string, string, string, error) + GetOperation() string + GetRefreshCache() bool +} + +var ( + volumeNameRe = regexp.MustCompile(`(?i)-volume\s+(\S+)`) + svmNameRe = regexp.MustCompile(`(?i)-vserver\s+(\S+)`) + newNameRe = regexp.MustCompile(`(?i)-newname\s+(\S+)`) + patchVolumeRe = regexp.MustCompile(`PATCH\s*/api/storage/volumes/([a-f0-9-]+)\s*:\s*.*?\{.*}`) + postVolumeRe = regexp.MustCompile(`POST\s*/api/storage/volumes\s*:\s*(\{.*})`) + deleteVolumeRe = regexp.MustCompile(`DELETE\s*/api/storage/volumes/([a-f0-9-]+)`) + postPrivateCliVolumeRe = regexp.MustCompile(`POST\s*/api/private/cli/volume/?\s*:\s*(\{.*})`) + postPrivateCliRenameRe = regexp.MustCompile(`POST\s*/api/private/cli/volume/rename/?\s*:\s*(\{.*})`) + deletePrivateCliVolumeRe = regexp.MustCompile(`DELETE\s*/api/private/cli/volume/?\s*:\s*(\{.*})`) + postApplicationRe = regexp.MustCompile(`POST\s*/api/application/applications\s*:\s*(?:\[\s*.*\s*])?\s*(\{.*})`) +) + +// VolumeWriteHandler handles volume write operations +type VolumeWriteHandler struct { + op string + refreshCache bool +} + +func (v VolumeWriteHandler) ExtractNames(input string, a *AuditLog) (string, string, string, error) { + err := a.RefreshVolumeCache(v.GetRefreshCache()) + if err != nil { + return "", "", "", err + } + volumeName := "" + svmName := "" + + volumeMatches := volumeNameRe.FindStringSubmatch(input) + if len(volumeMatches) > 1 { + volumeName = volumeMatches[1] + } + + svmMatches := svmNameRe.FindStringSubmatch(input) + if len(svmMatches) > 1 { + svmName = svmMatches[1] + } + + return volumeName, svmName, "", nil +} + +func (v VolumeWriteHandler) GetOperation() string { + return v.op +} + +func (v VolumeWriteHandler) GetRefreshCache() bool { + return v.refreshCache +} + +type VolumeRenameHandler struct { + op string + refreshCache bool +} + +func (v VolumeRenameHandler) ExtractNames(input string, a *AuditLog) (string, string, string, error) { + err := a.RefreshVolumeCache(v.GetRefreshCache()) + if err != nil { + return "", "", "", err + } + + svmName := "" + newName := "" + + svmMatches := svmNameRe.FindStringSubmatch(input) + if len(svmMatches) > 1 { + svmName = svmMatches[1] + } + + newNameMatches := newNameRe.FindStringSubmatch(input) + if len(newNameMatches) > 1 { + newName = newNameMatches[1] + } + + return newName, svmName, "", nil +} + +func (v VolumeRenameHandler) GetOperation() string { + return v.op +} + +func (v VolumeRenameHandler) GetRefreshCache() bool { + return v.refreshCache +} + +// VolumePatchHandler handles PATCH /api/storage/volumes +type VolumePatchHandler struct { + op string + refreshCache bool +} + +func (v VolumePatchHandler) ExtractNames(input string, a *AuditLog) (string, string, string, error) { + err := a.RefreshVolumeCache(v.GetRefreshCache()) + if err != nil { + return "", "", "", err + } + matches := patchVolumeRe.FindStringSubmatch(input) + if len(matches) < 2 { + return "", "", "", nil + } + + uuid := matches[1] + volumeInfo, exists := a.GetVolumeInfo(uuid) + if !exists { + return "", "", uuid, nil + } + + return volumeInfo.name, volumeInfo.svm, uuid, nil +} + +func (v VolumePatchHandler) GetOperation() string { + return v.op +} + +func (v VolumePatchHandler) GetRefreshCache() bool { + return v.refreshCache +} + +// VolumePostHandler handles POST /api/storage/volumes +type VolumePostHandler struct { + op string + refreshCache bool +} + +func (v VolumePostHandler) ExtractNames(input string, a *AuditLog) (string, string, string, error) { + err := a.RefreshVolumeCache(v.GetRefreshCache()) + if err != nil { + return "", "", "", err + } + matches := postVolumeRe.FindStringSubmatch(input) + if len(matches) < 2 { + return "", "", "", nil + } + + var payload struct { + SVM string `json:"svm"` + Name string `json:"name"` + } + if err := json.Unmarshal([]byte(matches[1]), &payload); err != nil { + return "", "", "", err + } + + return payload.Name, payload.SVM, "", nil +} + +func (v VolumePostHandler) GetOperation() string { + return v.op +} + +func (v VolumePostHandler) GetRefreshCache() bool { + return v.refreshCache +} + +// VolumeDeleteHandler handles DELETE /api/storage/volumes +type VolumeDeleteHandler struct { + op string + refreshCache bool +} + +func (v VolumeDeleteHandler) ExtractNames(input string, a *AuditLog) (string, string, string, error) { + err := a.RefreshVolumeCache(v.GetRefreshCache()) + if err != nil { + return "", "", "", err + } + matches := deleteVolumeRe.FindStringSubmatch(input) + if len(matches) < 2 { + return "", "", "", nil + } + + uuid := matches[1] + volumeInfo, exists := a.GetVolumeInfo(uuid) + if !exists { + return "", "", uuid, nil + } + + return volumeInfo.name, volumeInfo.svm, uuid, nil +} + +func (v VolumeDeleteHandler) GetOperation() string { + return v.op +} + +func (v VolumeDeleteHandler) GetRefreshCache() bool { + return v.refreshCache +} + +// VolumePrivateCliPostHandler handles POST /api/private/cli/volume +type VolumePrivateCliPostHandler struct { + op string + refreshCache bool +} + +func (v VolumePrivateCliPostHandler) ExtractNames(input string, a *AuditLog) (string, string, string, error) { + err := a.RefreshVolumeCache(v.GetRefreshCache()) + if err != nil { + return "", "", "", err + } + matches := postPrivateCliVolumeRe.FindStringSubmatch(input) + if len(matches) < 2 { + return "", "", "", nil + } + + var payload struct { + Vserver string `json:"vserver"` + Volume string `json:"volume"` + } + if err := json.Unmarshal([]byte(matches[1]), &payload); err != nil { + return "", "", "", err + } + + return payload.Volume, payload.Vserver, "", nil +} + +func (v VolumePrivateCliPostHandler) GetOperation() string { + return v.op +} + +func (v VolumePrivateCliPostHandler) GetRefreshCache() bool { + return v.refreshCache +} + +// VolumePrivateCliRenameHandler handles POST /api/private/cli/volume/rename +type VolumePrivateCliRenameHandler struct { + op string + refreshCache bool +} + +func (v VolumePrivateCliRenameHandler) ExtractNames(input string, a *AuditLog) (string, string, string, error) { + err := a.RefreshVolumeCache(v.GetRefreshCache()) + if err != nil { + return "", "", "", err + } + matches := postPrivateCliRenameRe.FindStringSubmatch(input) + if len(matches) < 2 { + return "", "", "", nil + } + + var payload struct { + Vserver string `json:"vserver"` + Volume string `json:"volume"` + NewName string `json:"newname"` + } + if err := json.Unmarshal([]byte(matches[1]), &payload); err != nil { + return "", "", "", err + } + + return payload.NewName, payload.Vserver, "", nil +} + +func (v VolumePrivateCliRenameHandler) GetOperation() string { + return v.op +} + +func (v VolumePrivateCliRenameHandler) GetRefreshCache() bool { + return v.refreshCache +} + +// VolumePrivateCliDeleteCliHandler handles DELETE /api/private/cli/volume +type VolumePrivateCliDeleteCliHandler struct { + op string + refreshCache bool +} + +func (v VolumePrivateCliDeleteCliHandler) ExtractNames(input string, a *AuditLog) (string, string, string, error) { + err := a.RefreshVolumeCache(v.GetRefreshCache()) + if err != nil { + return "", "", "", err + } + matches := deletePrivateCliVolumeRe.FindStringSubmatch(input) + if len(matches) < 2 { + return "", "", "", nil + } + + var payload struct { + Vserver string `json:"vserver"` + Volume string `json:"volume"` + } + if err := json.Unmarshal([]byte(matches[1]), &payload); err != nil { + return "", "", "", err + } + + return payload.Volume, payload.Vserver, "", nil +} + +func (v VolumePrivateCliDeleteCliHandler) GetOperation() string { + return v.op +} + +func (v VolumePrivateCliDeleteCliHandler) GetRefreshCache() bool { + return v.refreshCache +} + +type ApplicationPostHandler struct { + op string + refreshCache bool +} + +func (ap ApplicationPostHandler) ExtractNames(input string, a *AuditLog) (string, string, string, error) { + err := a.RefreshVolumeCache(ap.GetRefreshCache()) + if err != nil { + return "", "", "", err + } + matches := postApplicationRe.FindStringSubmatch(input) + if len(matches) < 2 { + return "", "", "", nil + } + + var payload struct { + Name string `json:"name"` + SVM struct { + Name string `json:"name"` + } `json:"svm"` + Template struct { + Name string `json:"name"` + } `json:"template"` + } + if err := json.Unmarshal([]byte(matches[1]), &payload); err != nil { + return "", "", "", err + } + + if payload.Template.Name != "nas" { + return "", "", "", nil + } + + return payload.Name, payload.SVM.Name, "", nil +} + +func (ap ApplicationPostHandler) GetOperation() string { + return ap.op +} + +func (ap ApplicationPostHandler) GetRefreshCache() bool { + return ap.refreshCache +} + +var volumeInputHandlers = map[*regexp.Regexp]VolumeHandler{ + regexp.MustCompile(`^volume create\s+`): VolumeWriteHandler{op: "create", refreshCache: true}, + regexp.MustCompile(`^volume modify\s+`): VolumeWriteHandler{op: "update"}, + regexp.MustCompile(`^volume delete\s+`): VolumeWriteHandler{op: "delete"}, + regexp.MustCompile(`^volume rename\s+`): VolumeRenameHandler{op: "rename"}, + postVolumeRe: VolumePostHandler{op: "create", refreshCache: true}, + patchVolumeRe: VolumePatchHandler{op: "update"}, + deleteVolumeRe: VolumeDeleteHandler{op: "delete"}, + postPrivateCliVolumeRe: VolumePrivateCliPostHandler{op: "create", refreshCache: true}, + postPrivateCliRenameRe: VolumePrivateCliRenameHandler{op: "update"}, + deletePrivateCliVolumeRe: VolumePrivateCliDeleteCliHandler{op: "delete"}, + postApplicationRe: ApplicationPostHandler{op: "create", refreshCache: true}, +} + +func (a *AuditLog) parseVolumeRecords(response []gjson.Result) { + mat := a.data + object := "volume" + for _, result := range response { + timestamp := result.Get("timestamp") + var auditTimeStamp int64 + if timestamp.Exists() { + t, err := time.Parse(time.RFC3339, timestamp.ClonedString()) + if err != nil { + a.SLogger.Error( + "Failed to load cluster date", + slogx.Err(err), + slog.String("date", timestamp.ClonedString()), + ) + continue + } + auditTimeStamp = t.Unix() + } + application := result.Get("application").ClonedString() + location := result.Get("location").ClonedString() + user := result.Get("user").ClonedString() + input := normalizeInput(result.Get("input").ClonedString()) + for re, handler := range volumeInputHandlers { + if !re.MatchString(input) { + continue + } + volume, svm, uuid, err := handler.ExtractNames(input, a) + if err != nil { + a.SLogger.Warn("error while parsing audit log", slogx.Err(err), slog.String("key", input)) + continue + } + if volume == "" || svm == "" { + if uuid == "" { + a.SLogger.Warn("failed to extract data", slog.String("input", input)) + continue + } + } + instanceKey := application + location + user + svm + volume + uuid + handler.GetOperation() + object + if instance := mat.GetInstance(instanceKey); instance != nil { + err = a.setLogMetric(mat, instance, float64(auditTimeStamp)) + if err != nil { + a.SLogger.Warn( + "Unable to set value on metric", + slogx.Err(err), + slog.String("metric", "log"), + ) + continue + } + } else { + instance, err = mat.NewInstance(instanceKey) + if err != nil { + a.SLogger.Warn("error while creating instance", slog.String("key", instanceKey)) + continue + } + if volume == "" && svm == "" { + instance.SetLabel("uuid", uuid) + } + instance.SetLabel("application", application) + instance.SetLabel("location", location) + instance.SetLabel("user", user) + instance.SetLabel("op", handler.GetOperation()) + instance.SetLabel("object", object) + instance.SetLabel("volume", volume) + instance.SetLabel("svm", svm) + err := a.setLogMetric(mat, instance, float64(auditTimeStamp)) + if err != nil { + a.SLogger.Warn("error while setting metric value", slogx.Err(err)) + return + } + } + } + } +} + +func normalizeInput(input string) string { + return replacer.Replace(input) +} diff --git a/cmd/collectors/rest/plugins/auditlog/auditlog_volume_test.go b/cmd/collectors/rest/plugins/auditlog/auditlog_volume_test.go new file mode 100644 index 000000000..1e508a032 --- /dev/null +++ b/cmd/collectors/rest/plugins/auditlog/auditlog_volume_test.go @@ -0,0 +1,237 @@ +package auditlog + +import ( + "testing" +) + +var auditlog = &AuditLog{ + AbstractPlugin: nil, + schedule: 0, + data: nil, + client: nil, + rootConfig: RootConfig{}, + lastFilterTimes: nil, + volumeCache: VolumeCache{}, +} + +func init() { + auditlog.InitVolumeCache() + auditlog.volumeCache.cache["123e4567-e89b-12d3-a456-426614174000"] = VolumeInfo{name: "testVolume", svm: "testSVM"} +} + +func TestVolumeWriteCreateHandler(t *testing.T) { + handler := VolumeWriteHandler{op: "create"} + inputs := []string{ + "volume create -vserver testSVM -volume testVolume -state online -policy default -size 200MB -aggregate umeng_aff300_aggr2", + "volume create -volume testVolume -size 200MB -vserver testSVM -state online -policy default -aggregate umeng_aff300_aggr2", + } + + for _, input := range inputs { + input = normalizeInput(input) + volume, svm, _, _ := handler.ExtractNames(input, auditlog) + if volume != "testVolume" || svm != "testSVM" { + t.Errorf("Input: %s, Expected volume: testVolume, svm: testSVM, got volume: %s, svm: %s", input, volume, svm) + } + if handler.GetOperation() != "create" { + t.Errorf("Expected operation: create, got: %s", handler.GetOperation()) + } + } +} + +func TestVolumeWriteModifyHandler(t *testing.T) { + handler := VolumeWriteHandler{op: "update"} + inputs := []string{ + "volume modify -vserver testSVM -volume testVolume -size 201MB -state online", + "volume modify -vserver testSVM -volume testVolume -size 201MB -state online", + "volume modify -vserver testSVM -size 201MB -state online -volume testVolume", + "volume modify -vserver testSVM -volume testVolume -state offline", + } + + for _, input := range inputs { + input = normalizeInput(input) + volume, svm, _, _ := handler.ExtractNames(input, auditlog) + if volume != "testVolume" || svm != "testSVM" { + t.Errorf("Input: %s, Expected volume: testVolume, svm: testSVM, got volume: %s, svm: %s", input, volume, svm) + } + if handler.GetOperation() != "update" { + t.Errorf("Expected operation: update, got: %s", handler.GetOperation()) + } + } +} + +func TestVolumeWriteRenameHandler(t *testing.T) { + handler := VolumeRenameHandler{op: "update"} + inputs := []string{ + "volume rename -vserver testSVM -volume testVolume -newname testVolume1", + "volume rename -vserver testSVM -volume testVolume -newname testVolume1", + } + + for _, input := range inputs { + input = normalizeInput(input) + volume, svm, _, _ := handler.ExtractNames(input, auditlog) + if volume != "testVolume1" || svm != "testSVM" { + t.Errorf("Input: %s, Expected volume: testVolume1, svm: testSVM, got volume: %s, svm: %s", input, volume, svm) + } + if handler.GetOperation() != "update" { + t.Errorf("Expected operation: update, got: %s", handler.GetOperation()) + } + } +} + +func TestVolumeWriteDeleteHandler(t *testing.T) { + handler := VolumeWriteHandler{op: "delete"} + inputs := []string{ + "volume create -volume testVolume -vserver testSVM", + "volume create -volume testVolume -vserver testSVM", + "volume create -volume\ttestVolume\t-vserver\ttestSVM", + "volume create -volume\ntestVolume\n-vserver\ntestSVM", + } + + for _, input := range inputs { + input = normalizeInput(input) + volume, svm, _, _ := handler.ExtractNames(input, auditlog) + if volume != "testVolume" || svm != "testSVM" { + t.Errorf("Input: %s, Expected volume: testVolume, svm: testSVM, got volume: %s, svm: %s", input, volume, svm) + } + if handler.GetOperation() != "delete" { + t.Errorf("Expected operation: delete, got: %s", handler.GetOperation()) + } + } +} + +func TestVolumePatchHandler(t *testing.T) { + handler := VolumePatchHandler{op: "update"} + inputs := []string{ + "PATCH /api/storage/volumes/123e4567-e89b-12d3-a456-426614174000 : {\"name\": \"testVolume\", \"size\": \"220MB\"}", + "PATCH /api/storage/volumes/123e4567-e89b-12d3-a456-426614174000 : {\"name\": \"testVolume\", \"size\": \"220MB\" }", + "PATCH /api/storage/volumes/123e4567-e89b-12d3-a456-426614174000 : [\"X-Dot-Client-App: SMv4\"] {\"state\":\"online\"}", + } + + for _, input := range inputs { + input = normalizeInput(input) + volume, svm, _, _ := handler.ExtractNames(input, auditlog) + if volume != "testVolume" || svm != "testSVM" { + t.Errorf("Input: %s, Expected volume: testVolume, svm: testSVM, got volume: %s, svm: %s", input, volume, svm) + } + if handler.GetOperation() != "update" { + t.Errorf("Expected operation: update, got: %s", handler.GetOperation()) + } + } +} + +func TestVolumePostHandler(t *testing.T) { + handler := VolumePostHandler{op: "create"} + inputs := []string{ + `POST /api/storage/volumes : {"svm":"testSVM","name":"testVolume"}`, + `POST /api/storage/volumes : {"svm": "testSVM", "name": "testVolume"}`, + } + + for _, input := range inputs { + input = normalizeInput(input) + volume, svm, _, _ := handler.ExtractNames(input, auditlog) + if volume != "testVolume" || svm != "testSVM" { + t.Errorf("Input: %s, Expected volume: testVolume, svm: testSVM, got volume: %s, svm: %s", input, volume, svm) + } + if handler.GetOperation() != "create" { + t.Errorf("Expected operation: create, got: %s", handler.GetOperation()) + } + } +} + +func TestVolumeDeleteHandler(t *testing.T) { + handler := VolumeDeleteHandler{op: "delete"} + inputs := []string{ + "DELETE /api/storage/volumes/123e4567-e89b-12d3-a456-426614174000", + "DELETE /api/storage/volumes/123e4567-e89b-12d3-a456-426614174000", + } + + for _, input := range inputs { + input = normalizeInput(input) + volume, svm, _, _ := handler.ExtractNames(input, auditlog) + if volume != "testVolume" || svm != "testSVM" { + t.Errorf("Input: %s, Expected volume: testVolume, svm: testSVM, got volume: %s, svm: %s", input, volume, svm) + } + if handler.GetOperation() != "delete" { + t.Errorf("Expected operation: delete, got: %s", handler.GetOperation()) + } + } +} + +func TestVolumePrivateCliPostHandler(t *testing.T) { + handler := VolumePrivateCliPostHandler{op: "create"} + inputs := []string{ + `POST /api/private/cli/volume : {"vserver":"testSVM","volume":"testVolume"}`, + `POST /api/private/cli/volume : {"vserver": "testSVM", "volume": "testVolume"}`, + `POST /api/private/cli/volume : { ^I^I\"vserver\": \"testSVM\", ^I^I\"volume\": \"testVolume\", ^I^I\"size\": \"200MB\", ^I^I\"aggregate\": \"umeng_aff300_aggr1\" ^I}`, + } + + for _, input := range inputs { + input = normalizeInput(input) + volume, svm, _, _ := handler.ExtractNames(input, auditlog) + if volume != "testVolume" || svm != "testSVM" { + t.Errorf("Input: %s, Expected volume: testVolume, svm: testSVM, got volume: %s, svm: %s", input, volume, svm) + } + if handler.GetOperation() != "create" { + t.Errorf("Expected operation: create, got: %s", handler.GetOperation()) + } + } +} + +func TestVolumePrivateCliRenameHandler(t *testing.T) { + handler := VolumePrivateCliRenameHandler{op: "update"} + inputs := []string{ + `POST /api/private/cli/volume/rename : {"vserver":"testSVM","volume":"testVolume","newname":"newTestVolume"}`, + `POST /api/private/cli/volume/rename : {"vserver": "testSVM", "volume": "testVolume", "newname": "newTestVolume"}`, + } + + for _, input := range inputs { + input = normalizeInput(input) + volume, svm, _, _ := handler.ExtractNames(input, auditlog) + if volume != "newTestVolume" || svm != "testSVM" { + t.Errorf("Input: %s, Expected volume: newTestVolume, svm: testSVM, got volume: %s, svm: %s", input, volume, svm) + } + if handler.GetOperation() != "update" { + t.Errorf("Expected operation: update, got: %s", handler.GetOperation()) + } + } +} + +func TestVolumePrivateCliDeleteCliHandler(t *testing.T) { + handler := VolumePrivateCliDeleteCliHandler{op: "delete"} + inputs := []string{ + `DELETE /api/private/cli/volume : {"vserver":"testSVM","volume":"testVolume"}`, + `DELETE /api/private/cli/volume : {"vserver": "testSVM", "volume": "testVolume"}`, + `DELETE /api/private/cli/volume : { ^I^I\"vserver\": \"testSVM\", ^I^I\"volume\": \"testVolume\" ^I}`, + } + + for _, input := range inputs { + input = normalizeInput(input) + volume, svm, _, _ := handler.ExtractNames(input, auditlog) + if volume != "testVolume" || svm != "testSVM" { + t.Errorf("Input: %s, Expected volume: testVolume, svm: testSVM, got volume: %s, svm: %s", input, volume, svm) + } + if handler.GetOperation() != "delete" { + t.Errorf("Expected operation: delete, got: %s", handler.GetOperation()) + } + } +} + +func TestApplicationPostHandler(t *testing.T) { + handler := ApplicationPostHandler{op: "create"} + inputs := []string{ + `POST /api/application/applications : ["X-Dot-Client-App: SMv4"] {"name":"testApp","svm":{"name":"testSVM"},"template":{"name":"nas"}}`, + `POST /api/application/applications : ["X-Dot-Client-App: SMv4"] {"name": "testApp", "svm": {"name": "testSVM"}, "template": {"name": "nas"}}`, + `POST /api/application/applications : {\"name\":\"testApp\",\"smart_container\":true,\"svm\":{\"name\":\"testSVM\"},\"nas\":{\"nfs_access\":[],\"cifs_access\":[],\"application_components\":[{\"name\":\"t1\",\"total_size\":1073741824,\"share_count\":1,\"storage_service\":{\"name\":\"performance\"},\"export_policy\":{\"id\":30064771074}}],\"protection_type\":{\"remote_rpo\":\"none\"}},\"template\":{\"name\":\"nas\"}}`, + } + + for _, input := range inputs { + input = normalizeInput(input) + volume, svm, _, _ := handler.ExtractNames(input, auditlog) + if volume != "testApp" || svm != "testSVM" { + t.Errorf("Input: %s, Expected volume: testApp, svm: testSVM, got volume: %s, svm: %s", input, volume, svm) + } + if handler.GetOperation() != "create" { + t.Errorf("Expected operation: create, got: %s", handler.GetOperation()) + } + } +} diff --git a/cmd/collectors/rest/rest.go b/cmd/collectors/rest/rest.go index 5e522569a..83f1a55ee 100644 --- a/cmd/collectors/rest/rest.go +++ b/cmd/collectors/rest/rest.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/netapp/harvest/v2/cmd/collectors" "github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/aggregate" + "github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/auditlog" "github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/certificate" "github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/cluster" "github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/clusterschedule" @@ -472,6 +473,8 @@ func (r *Rest) LoadPlugin(kind string, abc *plugin.AbstractPlugin) plugin.Plugin switch kind { case "Aggregate": return aggregate.New(abc) + case "AuditLog": + return auditlog.New(abc) case "Cluster": return cluster.New(abc) case "ClusterSchedule": diff --git a/cmd/tools/grafana/dashboard_test.go b/cmd/tools/grafana/dashboard_test.go index 188fd85eb..b8a3d57ed 100644 --- a/cmd/tools/grafana/dashboard_test.go +++ b/cmd/tools/grafana/dashboard_test.go @@ -1053,6 +1053,7 @@ func TestOnlyHighlightsExpanded(t *testing.T) { "cmode/health.json": 2, "cmode/power.json": 2, "storagegrid/fabricpool.json": 2, + "cmode/auditlog.json": 2, "cmode/nfsTroubleshooting.json": 3, } // count the number of expanded sections in the dashboard and ensure num expanded = 1 @@ -1555,13 +1556,24 @@ func checkDashboardTime(t *testing.T, path string, data []byte) { from := gjson.GetBytes(data, "time.from") to := gjson.GetBytes(data, "time.to") - fromWant := "now-3h" - toWant := "now" - if from.ClonedString() != fromWant { - t.Errorf("dashboard=%s time.from got=%s want=%s", dashPath, from.ClonedString(), fromWant) + expectedTimeRanges := map[string]struct { + from string + to string + }{ + "cmode/auditlog.json": {"now-24h", "now"}, + "default": {"now-3h", "now"}, } - if to.ClonedString() != toWant { - t.Errorf("dashboard=%s time.to got=%s want=%s", dashPath, to.ClonedString(), toWant) + + expected, exists := expectedTimeRanges[dashPath] + if !exists { + expected = expectedTimeRanges["default"] + } + + if from.ClonedString() != expected.from { + t.Errorf("dashboard=%s time.from got=%s want=%s", dashPath, from.ClonedString(), expected.from) + } + if to.ClonedString() != expected.to { + t.Errorf("dashboard=%s time.to got=%s want=%s", dashPath, to.ClonedString(), expected.to) } } @@ -1633,7 +1645,6 @@ func checkDescription(t *testing.T, path string, data []byte, count *int) { title := value.Get("title").ClonedString() panelType := value.Get("type").ClonedString() if slices.Contains(ignoreList, title) { - fmt.Printf(`dashboard=%s panel="%s" has different description\n`, dashPath, title) return } @@ -1658,11 +1669,11 @@ func checkDescription(t *testing.T, path string, data []byte, count *int) { t.Errorf(`dashboard=%s panel="%s" has many expressions %d`, dashPath, title, *count) } } - } else if !strings.HasPrefix(description, "$") && !strings.HasSuffix(description, ".") { + } else if !strings.HasPrefix(description, "$") && !strings.HasSuffix(strings.TrimSpace(description), ".") { // A few panels take their description text from a variable. // Those can be ignored. // Descriptions must end with a period (.) - t.Errorf(`dashboard=%s panel="%s" description should end with a period`, dashPath, title) + t.Errorf(`dashboard=%s panel="%s" description "%s" should end with a period`, dashPath, title, description) } }) } diff --git a/conf/rest/9.12.0/audit_log.yaml b/conf/rest/9.12.0/audit_log.yaml new file mode 100644 index 000000000..46c433c40 --- /dev/null +++ b/conf/rest/9.12.0/audit_log.yaml @@ -0,0 +1,12 @@ +name: AuditLog +query: api/cluster +object: audit + +counters: + - ^^uuid + - ^name + +plugins: + - AuditLog + +export_data: false \ No newline at end of file diff --git a/conf/rest/default.yaml b/conf/rest/default.yaml index 054f0f7f0..1f4d6bac4 100644 --- a/conf/rest/default.yaml +++ b/conf/rest/default.yaml @@ -11,6 +11,7 @@ schedule: objects: Aggregate: aggr.yaml AggregateEfficiency: aggr_efficiency.yaml +# AuditLog: audit_log.yaml # The CIFSSession template may slow down data collection due to a high number of metrics. # CIFSSession: cifs_session.yaml # CIFSShare: cifs_share.yaml diff --git a/grafana/dashboards/cmode/auditlog.json b/grafana/dashboards/cmode/auditlog.json new file mode 100644 index 000000000..d759aec52 --- /dev/null +++ b/grafana/dashboards/cmode/auditlog.json @@ -0,0 +1,722 @@ +{ + "__inputs": [ + { + "description": "", + "label": "Prometheus", + "name": "DS_PROMETHEUS", + "pluginId": "prometheus", + "pluginName": "Prometheus", + "type": "datasource" + } + ], + "__requires": [ + { + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "8.1.8" + }, + { + "id": "prometheus", + "name": "Prometheus", + "type": "datasource", + "version": "1.0.0" + }, + { + "id": "stat", + "name": "Stat", + "type": "panel", + "version": "" + }, + { + "id": "table", + "name": "Table", + "type": "panel", + "version": "" + }, + { + "id": "text", + "name": "Text", + "type": "panel", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "The ONTAP Changelog Monitor, tracks configuration modifications in volumes, SVMs, and nodes, is deactivated by default. To leverage this feature, one must enable the ChangeLog plugin within the Volume, SVM, and Node Templates.", + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "id": null, + "iteration": 1740047780335, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "cdot" + ], + "targetBlank": false, + "title": "Related Dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 303, + "panels": [], + "title": "Important Information", + "type": "row" + }, + { + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 305, + "options": { + "content": "
\n\nThis dashboard captures operations (create, update, delete) attempted on volumes via REST or ONTAP CLI commands. To use this dashboard, enable the AuditLog template. For more details, visit the [AuditLog documentation](https://github.com/NetApp/harvest/discussions/3478).\n", + "mode": "markdown" + }, + "pluginVersion": "8.1.8", + "type": "text" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 290, + "panels": [], + "title": "Volume Audit", + "type": "row" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "This change type indicates that Volume has been created.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 5 + }, + "id": 299, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "sum(changes(audit_log{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", object=~\"volume\", op=\"create\"}[$__range]) + 1) by (cluster, datacenter, object)", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Create", + "type": "stat" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "This change type indicates that Volume object has been updated.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 5 + }, + "id": 284, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "sum(changes(audit_log{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", object=~\"volume\", op=\"update\"}[$__range]) + 1) by (cluster, datacenter, object)", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Update", + "type": "stat" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "This change type indicates that Volume has been deleted.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 5 + }, + "id": 293, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "sum(changes(audit_log{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", object=~\"volume\", op=\"delete\"}[$__range]) + 1) by (cluster, datacenter, object)", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Delete", + "type": "stat" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "`Cluster Time:` The time on the cluster.\n\n`Object:` The name of the ONTAP object that was changed.\n\n`OP:` The type of change that was made (e.g., create, update, delete).\n\n`User`: The user who performed this action.\n\n`Location`: The IP address of the user.\n\n `Application`: The application used to perform the action. Possible values are `http` and `ssh`.\n\n`Count`: Number of times operation is performed.\n\n**Note:** Only REST and ONTAP CLI commands are included in the audit. ZAPI commands are not supported.\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "filterable": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "ClusterTime" + }, + "properties": [ + { + "id": "unit", + "value": "dateTimeAsIso" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cluster Time" + }, + "properties": [ + { + "id": "custom.align", + "value": "left" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cluster" + }, + "properties": [ + { + "id": "displayName", + "value": "Cluster" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-cluster/ontap-cluster?orgId=1&${Datacenter:queryparam}&${__url_time_range}&var-Cluster=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "datacenter" + }, + "properties": [ + { + "id": "displayName", + "value": "Datacenter" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-datacenter/ontap-datacenter?orgId=1&${__url_time_range}&var-Datacenter=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "node" + }, + "properties": [ + { + "id": "displayName", + "value": "Node" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-node/ontap-node?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${__url_time_range}&var-Node=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "svm" + }, + "properties": [ + { + "id": "displayName", + "value": "SVM" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-svm/ontap-svm?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${__url_time_range}&var-SVM=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "volume" + }, + "properties": [ + { + "id": "displayName", + "value": "Volume" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-volume/ontap-volume?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${SVM:queryparam}&${__url_time_range}&var-Volume=${__value.raw}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 295, + "options": { + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "label_join(last_over_time(audit_log{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", object=~\"volume\"}[$__range]), \"unique_id\", \"-\", \"datacenter\", \"cluster\", \"object\", \"volume\",\"uuid\",\"op\",\"application\",\"location\", \"user\")", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "exemplar": false, + "expr": "label_join(sum(changes(audit_log{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", object=~\"volume\"}[$__range]) + 1) by (cluster, datacenter, object, volume, uuid, op,application, location, user), \"unique_id\", \"-\", \"datacenter\", \"cluster\", \"object\", \"volume\",\"uuid\",\"op\",\"application\",\"location\",\"user\")", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "title": "Volume Changes", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "unique_id" + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "(.*) 1$", + "renamePattern": "$1" + } + }, + { + "id": "calculateField", + "options": { + "alias": "ClusterTime", + "binary": { + "left": "Value #A", + "operator": "*", + "reducer": "sum", + "right": "1000" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": false + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "application", + "cluster", + "datacenter", + "location", + "op", + "svm", + "user", + "volume", + "Value #A", + "ClusterTime", + "Value #B" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time 2": true, + "Time 3": true, + "Value #A": true, + "application 2": true, + "cluster 2": true, + "datacenter 2": true, + "location 2": true, + "object 2": true, + "op 2": true, + "user 2": true, + "uuid 2": true, + "volume 2": true + }, + "indexByName": { + "ClusterTime": 0, + "Value #A": 16, + "application": 8, + "application 2": 9, + "cluster": 2, + "cluster 2": 10, + "datacenter": 1, + "datacenter 2": 11, + "location": 7, + "location 2": 12, + "op": 5, + "op 2": 13, + "svm": 3, + "user": 6, + "user 2": 14, + "volume": 4, + "volume 2": 15 + }, + "renameByName": { + "ClusterTime": "Cluster Time", + "Value #B": "Count", + "application": "Application", + "op": "OP", + "svm": "", + "user": "User" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Cluster Time" + } + ] + } + } + ], + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 30, + "style": "dark", + "tags": [ + "harvest", + "ontap", + "cdot" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": null, + "error": null, + "hide": 2, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "label_values(cluster_new_status{system_type!=\"7mode\"},datacenter)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "Datacenter", + "options": [], + "query": { + "query": "label_values(cluster_new_status{system_type!=\"7mode\"},datacenter)", + "refId": "Prometheus-Datacenter-Variable-Query" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "label_values(cluster_new_status{system_type!=\"7mode\",datacenter=~\"$Datacenter\"},cluster)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "Cluster", + "options": [], + "query": { + "query": "label_values(cluster_new_status{system_type!=\"7mode\",datacenter=~\"$Datacenter\"},cluster)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "ONTAP: AuditLog", + "uid": "cdot-auditlog", + "version": 1 +} diff --git a/grafana/dashboards/cmode/changelogMonitor.json b/grafana/dashboards/cmode/changelogMonitor.json index 8c3b380c0..3395491dc 100644 --- a/grafana/dashboards/cmode/changelogMonitor.json +++ b/grafana/dashboards/cmode/changelogMonitor.json @@ -65,7 +65,7 @@ "gnetId": null, "graphTooltip": 1, "id": null, - "iteration": 1696405583422, + "iteration": 1739793314493, "links": [ { "asDropdown": true, @@ -126,7 +126,7 @@ "panels": [ { "datasource": "${DS_PROMETHEUS}", - "description": "This change type indicates that an existing ONTAP object has been updated.", + "description": "This change type indicates that a new ONTAP object has been created.", "fieldConfig": { "defaults": { "color": { @@ -151,9 +151,9 @@ "h": 5, "w": 8, "x": 0, - "y": 1 + "y": 4 }, - "id": 291, + "id": 285, "options": { "colorMode": "value", "graphMode": "area", @@ -173,19 +173,20 @@ "targets": [ { "exemplar": false, - "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"node\",op=\"update\"}[$__range]))", + "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"node\",op=\"create\"}[$__range]))", + "format": "time_series", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], - "title": "Update", + "title": "Create", "type": "stat" }, { "datasource": "${DS_PROMETHEUS}", - "description": "This change type indicates that a new ONTAP object has been created.", + "description": "This change type indicates that an existing ONTAP object has been updated.", "fieldConfig": { "defaults": { "color": { @@ -210,9 +211,9 @@ "h": 5, "w": 8, "x": 8, - "y": 1 + "y": 4 }, - "id": 285, + "id": 291, "options": { "colorMode": "value", "graphMode": "area", @@ -232,15 +233,14 @@ "targets": [ { "exemplar": false, - "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"node\",op=\"create\"}[$__range]))", - "format": "time_series", + "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"node\",op=\"update\"}[$__range]))", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], - "title": "Create", + "title": "Update", "type": "stat" }, { @@ -270,7 +270,7 @@ "h": 5, "w": 8, "x": 16, - "y": 1 + "y": 4 }, "id": 286, "options": { @@ -424,7 +424,7 @@ "h": 8, "w": 24, "x": 0, - "y": 6 + "y": 9 }, "id": 288, "options": { @@ -540,7 +540,7 @@ "panels": [ { "datasource": "${DS_PROMETHEUS}", - "description": "This change type indicates that an existing ONTAP object has been updated.", + "description": "This change type indicates that a new ONTAP object has been created.", "fieldConfig": { "defaults": { "color": { @@ -565,9 +565,9 @@ "h": 5, "w": 8, "x": 0, - "y": 2 + "y": 5 }, - "id": 298, + "id": 292, "options": { "colorMode": "value", "graphMode": "area", @@ -587,19 +587,20 @@ "targets": [ { "exemplar": false, - "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"svm\",op=\"update\"}[$__range]))", + "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"svm\",op=\"create\"}[$__range]))", + "format": "time_series", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], - "title": "Update", + "title": "Create", "type": "stat" }, { "datasource": "${DS_PROMETHEUS}", - "description": "This change type indicates that a new ONTAP object has been created.", + "description": "This change type indicates that an existing ONTAP object has been updated.", "fieldConfig": { "defaults": { "color": { @@ -624,9 +625,9 @@ "h": 5, "w": 8, "x": 8, - "y": 2 + "y": 5 }, - "id": 292, + "id": 298, "options": { "colorMode": "value", "graphMode": "area", @@ -646,15 +647,14 @@ "targets": [ { "exemplar": false, - "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"svm\",op=\"create\"}[$__range]))", - "format": "time_series", + "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"svm\",op=\"update\"}[$__range]))", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], - "title": "Create", + "title": "Update", "type": "stat" }, { @@ -684,7 +684,7 @@ "h": 5, "w": 8, "x": 16, - "y": 2 + "y": 5 }, "id": 300, "options": { @@ -860,7 +860,7 @@ "h": 8, "w": 24, "x": 0, - "y": 7 + "y": 10 }, "id": 301, "options": { @@ -980,7 +980,7 @@ "panels": [ { "datasource": "${DS_PROMETHEUS}", - "description": "This change type indicates that an existing ONTAP object has been updated.", + "description": "This change type indicates that a new ONTAP object has been created.", "fieldConfig": { "defaults": { "color": { @@ -1005,9 +1005,9 @@ "h": 5, "w": 8, "x": 0, - "y": 3 + "y": 6 }, - "id": 284, + "id": 299, "options": { "colorMode": "value", "graphMode": "area", @@ -1027,19 +1027,20 @@ "targets": [ { "exemplar": false, - "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"volume\",op=\"update\"}[$__range]))", + "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"volume\",op=\"create\"}[$__range]))", + "format": "time_series", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], - "title": "Update", + "title": "Create", "type": "stat" }, { "datasource": "${DS_PROMETHEUS}", - "description": "This change type indicates that a new ONTAP object has been created.", + "description": "This change type indicates that an existing ONTAP object has been updated.", "fieldConfig": { "defaults": { "color": { @@ -1064,9 +1065,9 @@ "h": 5, "w": 8, "x": 8, - "y": 3 + "y": 6 }, - "id": 299, + "id": 284, "options": { "colorMode": "value", "graphMode": "area", @@ -1086,15 +1087,14 @@ "targets": [ { "exemplar": false, - "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"volume\",op=\"create\"}[$__range]))", - "format": "time_series", + "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"volume\",op=\"update\"}[$__range]))", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], - "title": "Create", + "title": "Update", "type": "stat" }, { @@ -1124,7 +1124,7 @@ "h": 5, "w": 8, "x": 16, - "y": 3 + "y": 6 }, "id": 293, "options": { @@ -1322,7 +1322,7 @@ "h": 8, "w": 24, "x": 0, - "y": 8 + "y": 11 }, "id": 295, "options": { diff --git a/integration/test/dashboard_json_test.go b/integration/test/dashboard_json_test.go index e5e5aaf5b..95dc58b95 100644 --- a/integration/test/dashboard_json_test.go +++ b/integration/test/dashboard_json_test.go @@ -67,6 +67,7 @@ var restCounterMap = map[string]struct{}{ var excludeCounters = []string{ "aggr_physical_", "change_log", + "audit_log", "cifs_session", "cluster_peer", "efficiency_savings",