From 832b310d80d564304190a8bb0f5e98536828ace6 Mon Sep 17 00:00:00 2001 From: Deluan Date: Thu, 26 Dec 2024 17:28:40 -0500 Subject: [PATCH 1/9] Add summary --- db.go | 28 +++++++++-- go.mod | 20 ++++++-- go.sum | 38 ++++++++++++++- handler.go | 3 +- main.go | 23 ++++++--- summary.go | 134 +++++++++++++++++++++++++++++++++++++++++++++++++++++ tasks.go | 22 +++++++++ 7 files changed, 251 insertions(+), 17 deletions(-) create mode 100644 summary.go create mode 100644 tasks.go diff --git a/db.go b/db.go index 0fc8740..304e95e 100644 --- a/db.go +++ b/db.go @@ -27,14 +27,21 @@ func openDB(fileName string) (*sql.DB, error) { return nil, err } - // Create table if not exists + // Create schema if not exists createTableQuery := ` CREATE TABLE IF NOT EXISTS insights ( id VARCHAR NOT NULL, time DATETIME default CURRENT_TIMESTAMP, data JSONB, PRIMARY KEY (id, time) -);` +); +CREATE TABLE IF NOT EXISTS summary ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + time DATETIME UNIQUE, + data JSONB +); +CREATE INDEX IF NOT EXISTS idx_summary_time ON summary (time); +` _, err = db.Exec(createTableQuery) if err != nil { return nil, err @@ -44,14 +51,25 @@ CREATE TABLE IF NOT EXISTS insights ( return db, nil } -func saveToDB(db *sql.DB, data insights.Data) error { +func saveReport(db *sql.DB, data insights.Data, t time.Time) error { dataJSON, err := json.Marshal(data) if err != nil { return err } - query := `INSERT INTO insights (id, data) VALUES (?, ?)` - _, err = db.Exec(query, data.InsightsID, dataJSON) + query := `INSERT INTO insights (id, data, time) VALUES (?, ?, ?)` + _, err = db.Exec(query, data.InsightsID, dataJSON, t.Format("2006-01-02 15:04:05")) + return err +} + +func saveSummary(db *sql.DB, summary Summary, t time.Time) error { + summaryJSON, err := json.Marshal(summary) + if err != nil { + return err + } + + query := `INSERT INTO summary (time, data) VALUES (?, ?) ON CONFLICT(time) DO UPDATE SET data=?` + _, err = db.Exec(query, t.Format("2006-01-02 15:04:05"), summaryJSON, summaryJSON) return err } diff --git a/go.mod b/go.mod index c998ce8..9844b3d 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,26 @@ module github.com/navidrome/insights -go 1.23.3 +go 1.23.4 require ( github.com/go-chi/chi/v5 v5.2.0 github.com/go-chi/httprate v0.14.1 github.com/mattn/go-sqlite3 v1.14.24 - github.com/navidrome/navidrome v0.53.4-0.20241219230533-906ac635c296 + github.com/navidrome/navidrome v0.54.2 + github.com/onsi/ginkgo/v2 v2.22.0 + github.com/onsi/gomega v1.36.1 + github.com/robfig/cron/v3 v3.0.1 ) -require github.com/cespare/xxhash/v2 v2.3.0 // indirect +require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect + golang.org/x/net v0.32.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect + golang.org/x/tools v0.27.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum index c3dbe1c..05f20e5 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,46 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-chi/chi/v5 v5.2.0 h1:Aj1EtB0qR2Rdo2dG4O94RIU35w2lvQSj6BRA4+qwFL0= github.com/go-chi/chi/v5 v5.2.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-chi/httprate v0.14.1 h1:EKZHYEZ58Cg6hWcYzoZILsv7ppb46Wt4uQ738IRtpZs= github.com/go-chi/httprate v0.14.1/go.mod h1:TUepLXaz/pCjmCtf/obgOQJ2Sz6rC8fSf5cAt5cnTt0= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/navidrome/navidrome v0.53.4-0.20241219230533-906ac635c296 h1:xVDyE7lmkEUSNfz9vWHYueY35Ao/QxJEAFnGghOWtLA= -github.com/navidrome/navidrome v0.53.4-0.20241219230533-906ac635c296/go.mod h1:4uSM4pHPlsvdjVkjb9C98nLK38t3ph+zz1zjOCSKd4I= +github.com/navidrome/navidrome v0.54.2 h1:tDeiV6MHW/GuVh2Vyk/kQ8E24AIcwJZhjmeNLfXCZhM= +github.com/navidrome/navidrome v0.54.2/go.mod h1:4uSM4pHPlsvdjVkjb9C98nLK38t3ph+zz1zjOCSKd4I= +github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= +github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= +github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= +golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/tools v0.27.0 h1:qEKojBykQkQ4EynWy4S8Weg69NumxKdn40Fce3uc/8o= +golang.org/x/tools v0.27.0/go.mod h1:sUi0ZgbwW9ZPAq26Ekut+weQPR5eIM6GQLQ1Yjm1H0Q= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/handler.go b/handler.go index 0159468..afe1dd6 100644 --- a/handler.go +++ b/handler.go @@ -5,6 +5,7 @@ import ( "errors" "log" "net/http" + "time" "github.com/navidrome/navidrome/core/metrics/insights" ) @@ -25,7 +26,7 @@ func handler(db *sql.DB) http.HandlerFunc { return } - err = saveToDB(db, data) + err = saveReport(db, data, time.Now()) if err != nil { log.Printf("Error handling request: %s", err.Error()) w.WriteHeader(http.StatusInternalServerError) diff --git a/main.go b/main.go index e901e2b..257d28a 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "database/sql" "log" "net/http" @@ -10,23 +11,33 @@ import ( "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" "github.com/go-chi/httprate" + "github.com/robfig/cron/v3" ) -func dataCleaner(db *sql.DB) { - for { - log.Print("Cleaning old data") - _ = purgeOldEntries(db) - time.Sleep(24 * time.Hour) +func startTasks(ctx context.Context, db *sql.DB) error { + c := cron.New() + _, err := c.AddFunc("1 0 * * *", summarize(ctx, db)) + if err != nil { + return err + } + _, err = c.AddFunc("5 0 * * *", cleanup(ctx, db)) + if err != nil { + return err } + c.Start() + return nil } func main() { + ctx := context.Background() db, err := openDB("insights.db") if err != nil { log.Fatal(err) } - go dataCleaner(db) + if err := startTasks(ctx, db); err != nil { + log.Fatal(err) + } r := chi.NewRouter() r.Use(middleware.RealIP) diff --git a/summary.go b/summary.go new file mode 100644 index 0000000..36260cd --- /dev/null +++ b/summary.go @@ -0,0 +1,134 @@ +package main + +import ( + "database/sql" + "encoding/json" + "fmt" + "iter" + "log" + "strings" + "time" + + "github.com/navidrome/navidrome/core/metrics/insights" + "golang.org/x/text/cases" + "golang.org/x/text/language" +) + +type Summary struct { + Versions map[string]uint64 + OS map[string]uint64 + Players map[string]uint64 + Users map[string]uint64 + Tracks map[string]uint64 +} + +func summarizeData(db *sql.DB, date time.Time) error { + rows, err := selectData(db, date) + if err != nil { + log.Printf("Error selecting data: %s", err) + return err + } + summary := Summary{ + Versions: make(map[string]uint64), + OS: make(map[string]uint64), + Players: make(map[string]uint64), + Users: make(map[string]uint64), + Tracks: make(map[string]uint64), + } + for data := range rows { + // Summarize data here + summary.Versions[mapVersion(data)]++ + summary.OS[mapOS(data)]++ + mapPlayers(data, summary.Players) + mapToBins(data.Library.ActiveUsers, userBins, summary.Users) + mapToBins(data.Library.Tracks, trackBins, summary.Tracks) + } + // Save summary to database + err = saveSummary(db, summary, date) + if err != nil { + log.Printf("Error saving summary: %s", err) + return err + } + return err +} + +func mapVersion(data insights.Data) string { return data.Version } + +var trackBins = []int64{0, 1, 100, 500, 1000, 5000, 10000, 20000, 50000, 100000, 500000, 1000000} +var userBins = []int64{0, 1, 5, 10, 20, 50, 100, 200, 500, 1000} + +func mapToBins(count int64, bins []int64, counters map[string]uint64) { + for i := range bins { + bin := bins[len(bins)-1-i] + if count >= bin { + counters[fmt.Sprintf("%d", bin)]++ + return + } + } +} + +var caser = cases.Title(language.Und) + +func mapOS(data insights.Data) string { + os := func() string { + switch data.OS.Type { + case "darwin": + return "macOS" + case "linux": + if data.OS.Containerized { + return "Linux (containerized)" + } + return "Linux" + default: + s := strings.Replace(data.OS.Type, "bsd", "BSD", -1) + return caser.String(s) + } + }() + return os + " - " + data.OS.Arch +} + +func mapPlayers(data insights.Data, players map[string]uint64) { + for p, count := range data.Library.ActivePlayers { + players[p] = uint64(count) + } +} + +func selectData(db *sql.DB, date time.Time) (iter.Seq[insights.Data], error) { + query := fmt.Sprintf(` +SELECT i1.id, i1.time, i1.data +FROM insights i1 +INNER JOIN ( + SELECT id, MAX(time) as max_time + FROM insights + WHERE time >= date('%[1]s', '-1 day') AND time < date('%[1]s') + GROUP BY id +) i2 ON i1.id = i2.id AND i1.time = i2.max_time +WHERE i1.time >= date('%[1]s', '-1 day') AND time < date('%[1]s') +ORDER BY i1.id, i1.time DESC;`, date.Format("2006-01-02")) + rows, err := db.Query(query) + if err != nil { + return nil, fmt.Errorf("querying data: %w", err) + } + return func(yield func(insights.Data) bool) { + defer rows.Close() + for rows.Next() { + var j string + var id string + var t time.Time + err := rows.Scan(&id, &t, &j) + if err != nil { + log.Printf("Error scanning row: %s", err) + return + } + var data insights.Data + err = json.Unmarshal([]byte(j), &data) + if err != nil { + log.Printf("Error unmarshalling data: %s", err) + return + } + if !yield(data) { + return + } + } + }, nil +} diff --git a/tasks.go b/tasks.go new file mode 100644 index 0000000..3f4df61 --- /dev/null +++ b/tasks.go @@ -0,0 +1,22 @@ +package main + +import ( + "context" + "database/sql" + "log" + "time" +) + +func cleanup(_ context.Context, db *sql.DB) func() { + return func() { + log.Print("Cleaning old data") + _ = purgeOldEntries(db) + } +} + +func summarize(_ context.Context, db *sql.DB) func() { + return func() { + log.Print("Summarizing data") + _ = summarizeData(db, time.Now()) + } +} From cff13e137c05e0bde901eadc547488961dd981fe Mon Sep 17 00:00:00 2001 From: Deluan Date: Thu, 26 Dec 2024 17:38:25 -0500 Subject: [PATCH 2/9] Fix lint --- .golangci.yml | 4 ++++ summary.go | 11 ++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 19dcfd5..3dce60d 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -24,3 +24,7 @@ linters: - unconvert - unused - whitespace +linters-settings: + gosec: + excludes: + - G115 # Can't check context, where the warning is clearly a false positive. See discussion in https://github.com/securego/gosec/pull/1149 diff --git a/summary.go b/summary.go index 36260cd..8896dc1 100644 --- a/summary.go +++ b/summary.go @@ -94,18 +94,19 @@ func mapPlayers(data insights.Data, players map[string]uint64) { } func selectData(db *sql.DB, date time.Time) (iter.Seq[insights.Data], error) { - query := fmt.Sprintf(` + query := ` SELECT i1.id, i1.time, i1.data FROM insights i1 INNER JOIN ( SELECT id, MAX(time) as max_time FROM insights - WHERE time >= date('%[1]s', '-1 day') AND time < date('%[1]s') + WHERE time >= date(?, '-1 day') AND time < date(?) GROUP BY id ) i2 ON i1.id = i2.id AND i1.time = i2.max_time -WHERE i1.time >= date('%[1]s', '-1 day') AND time < date('%[1]s') -ORDER BY i1.id, i1.time DESC;`, date.Format("2006-01-02")) - rows, err := db.Query(query) +WHERE i1.time >= date(?, '-1 day') AND time < date(?) +ORDER BY i1.id, i1.time DESC;` + d := date.Format("2006-01-02") + rows, err := db.Query(query, d, d, d, d) if err != nil { return nil, fmt.Errorf("querying data: %w", err) } From 11d8eff56845760219cdc277a04e5cc3fb96b8c0 Mon Sep 17 00:00:00 2001 From: Deluan Date: Thu, 26 Dec 2024 23:45:49 -0500 Subject: [PATCH 3/9] Fix summary --- main.go | 4 +- summary.go | 71 ++++++++++++++++++++++-------- summary_test.go | 114 ++++++++++++++++++++++++++++++++++++++++++++++++ tasks.go | 7 ++- 4 files changed, 174 insertions(+), 22 deletions(-) create mode 100644 summary_test.go diff --git a/main.go b/main.go index 257d28a..562ee49 100644 --- a/main.go +++ b/main.go @@ -16,7 +16,7 @@ import ( func startTasks(ctx context.Context, db *sql.DB) error { c := cron.New() - _, err := c.AddFunc("1 0 * * *", summarize(ctx, db)) + _, err := c.AddFunc("1 * * * *", summarize(ctx, db)) if err != nil { return err } @@ -39,6 +39,8 @@ func main() { log.Fatal(err) } + summarize(ctx, db)() + r := chi.NewRouter() r.Use(middleware.RealIP) r.Use(middleware.Logger) diff --git a/summary.go b/summary.go index 8896dc1..5263acd 100644 --- a/summary.go +++ b/summary.go @@ -6,6 +6,7 @@ import ( "fmt" "iter" "log" + "regexp" "strings" "time" @@ -15,11 +16,12 @@ import ( ) type Summary struct { - Versions map[string]uint64 - OS map[string]uint64 - Players map[string]uint64 - Users map[string]uint64 - Tracks map[string]uint64 + Versions map[string]uint64 `json:"versions,omitempty"` + OS map[string]uint64 `json:"OS,omitempty"` + PlayerTypes map[string]uint64 `json:"playerTypes,omitempty"` + Players map[string]uint64 `json:"players,omitempty"` + Users map[string]uint64 `json:"users,omitempty"` + Tracks map[string]uint64 `json:"tracks,omitempty"` } func summarizeData(db *sql.DB, date time.Time) error { @@ -29,18 +31,20 @@ func summarizeData(db *sql.DB, date time.Time) error { return err } summary := Summary{ - Versions: make(map[string]uint64), - OS: make(map[string]uint64), - Players: make(map[string]uint64), - Users: make(map[string]uint64), - Tracks: make(map[string]uint64), + Versions: make(map[string]uint64), + OS: make(map[string]uint64), + PlayerTypes: make(map[string]uint64), + Players: make(map[string]uint64), + Users: make(map[string]uint64), + Tracks: make(map[string]uint64), } for data := range rows { // Summarize data here summary.Versions[mapVersion(data)]++ summary.OS[mapOS(data)]++ - mapPlayers(data, summary.Players) - mapToBins(data.Library.ActiveUsers, userBins, summary.Users) + summary.Users[fmt.Sprintf("%d", data.Library.ActiveUsers)]++ + totalPlayers := mapPlayerTypes(data, summary.PlayerTypes) + summary.Players[fmt.Sprintf("%d", totalPlayers)]++ mapToBins(data.Library.Tracks, trackBins, summary.Tracks) } // Save summary to database @@ -55,7 +59,6 @@ func summarizeData(db *sql.DB, date time.Time) error { func mapVersion(data insights.Data) string { return data.Version } var trackBins = []int64{0, 1, 100, 500, 1000, 5000, 10000, 20000, 50000, 100000, 500000, 1000000} -var userBins = []int64{0, 1, 5, 10, 20, 50, 100, 200, 500, 1000} func mapToBins(count int64, bins []int64, counters map[string]uint64) { for i := range bins { @@ -80,17 +83,47 @@ func mapOS(data insights.Data) string { } return "Linux" default: - s := strings.Replace(data.OS.Type, "bsd", "BSD", -1) - return caser.String(s) + s := caser.String(data.OS.Type) + return strings.Replace(s, "bsd", "BSD", -1) } }() return os + " - " + data.OS.Arch } -func mapPlayers(data insights.Data, players map[string]uint64) { +var playersTypes = map[*regexp.Regexp]string{ + regexp.MustCompile("NavidromeUI.*"): "NavidromeUI", + regexp.MustCompile("supersonic"): "Supersonic", + regexp.MustCompile("feishin_"): "", // Discard (old version) + regexp.MustCompile("audioling"): "Audioling", + regexp.MustCompile("playSub.*"): "play:Sub", + regexp.MustCompile("eu.callcc.audrey"): "audrey", + regexp.MustCompile("DSubCC"): "", // Discard (chromecast) + regexp.MustCompile(`bonob\+.*`): "", // Discard (transcodings) + regexp.MustCompile("https?://airsonic.*"): "Airsonic Refix", + regexp.MustCompile("multi-scrobbler.*"): "Multi-Scrobbler", + regexp.MustCompile("SubMusic.*"): "SubMusic", +} + +func mapPlayerTypes(data insights.Data, players map[string]uint64) int64 { + seen := map[string]uint64{} for p, count := range data.Library.ActivePlayers { - players[p] = uint64(count) + for r, t := range playersTypes { + if r.MatchString(p) { + p = t + break + } + } + if p != "" { + v := seen[p] + seen[p] = max(v, uint64(count)) + } + } + var total int64 + for k, v := range seen { + total += int64(v) + players[k] += v } + return total } func selectData(db *sql.DB, date time.Time) (iter.Seq[insights.Data], error) { @@ -100,10 +133,10 @@ FROM insights i1 INNER JOIN ( SELECT id, MAX(time) as max_time FROM insights - WHERE time >= date(?, '-1 day') AND time < date(?) + WHERE time >= date(?) AND time < date(?, '+1 day') GROUP BY id ) i2 ON i1.id = i2.id AND i1.time = i2.max_time -WHERE i1.time >= date(?, '-1 day') AND time < date(?) +WHERE i1.time >= date(?) AND time < date(?, '+1 day') ORDER BY i1.id, i1.time DESC;` d := date.Format("2006-01-02") rows, err := db.Query(query, d, d, d, d) diff --git a/summary_test.go b/summary_test.go new file mode 100644 index 0000000..a9ae16e --- /dev/null +++ b/summary_test.go @@ -0,0 +1,114 @@ +package main + +import ( + "maps" + "slices" + "testing" + + "github.com/navidrome/navidrome/core/metrics/insights" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestSummary(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Insights Suite") +} + +var _ = Describe("Summary", func() { + Describe("mapToBins", func() { + var counters map[string]uint64 + var testBins = []int64{0, 1, 5, 10, 20, 50, 100, 200, 500, 1000} + + BeforeEach(func() { + counters = make(map[string]uint64) + }) + + It("should map count to the correct bin", func() { + mapToBins(0, testBins, counters) + Expect(counters["0"]).To(Equal(uint64(1))) + + mapToBins(1, testBins, counters) + Expect(counters["1"]).To(Equal(uint64(1))) + + mapToBins(10, testBins, counters) + Expect(counters["10"]).To(Equal(uint64(1))) + + mapToBins(101, testBins, counters) + Expect(counters["100"]).To(Equal(uint64(1))) + + mapToBins(1000, testBins, counters) + Expect(counters["1000"]).To(Equal(uint64(1))) + }) + + It("should map count to the highest bin if count exceeds all bins", func() { + mapToBins(2000, testBins, counters) + Expect(counters["1000"]).To(Equal(uint64(1))) + }) + + It("should increment the correct bin count", func() { + mapToBins(5, testBins, counters) + mapToBins(5, testBins, counters) + Expect(counters["5"]).To(Equal(uint64(2))) + }) + + It("should handle empty bins array", func() { + mapToBins(5, []int64{}, counters) + Expect(counters).To(BeEmpty()) + }) + }) + + DescribeTable("mapOS", + func(expected string, data insights.Data) { + Expect(mapOS(data)).To(Equal(expected)) + }, + Entry("should map darwin to macOS", "macOS - x86_64", insights.Data{OS: insightsOS{Type: "darwin", Arch: "x86_64"}}), + Entry("should map linux to Linux", "Linux - x86_64", insights.Data{OS: insightsOS{Type: "linux", Arch: "x86_64"}}), + Entry("should map containerized linux to Linux (containerized)", "Linux (containerized) - x86_64", insights.Data{OS: insightsOS{Type: "linux", Containerized: true, Arch: "x86_64"}}), + Entry("should map bsd to BSD", "FreeBSD - x86_64", insights.Data{OS: insightsOS{Type: "freebsd", Arch: "x86_64"}}), + Entry("should map unknown OS types", "Unknown - x86_64", insights.Data{OS: insightsOS{Type: "unknown", Arch: "x86_64"}}), + ) + DescribeTable("mapPlayerTypes", + func(data insights.Data, expected map[string]uint64) { + players := make(map[string]uint64) + c := mapPlayerTypes(data, players) + Expect(players).To(Equal(expected)) + values := slices.Collect(maps.Values(expected)) + var total uint64 + for _, v := range values { + total += v + } + Expect(c).To(Equal(int64(total))) + }, + Entry("Feishin player", insights.Data{Library: insightsLibrary{ActivePlayers: map[string]int64{"feishin_": 1, "Feishin": 1}}}, map[string]uint64{"Feishin": 1}), + Entry("NavidromeUI player", insights.Data{Library: insightsLibrary{ActivePlayers: map[string]int64{"NavidromeUI_1.0": 2}}}, map[string]uint64{"NavidromeUI": 2}), + Entry("play:Sub player", insights.Data{Library: insightsLibrary{ActivePlayers: map[string]int64{"playSub_iPhone11": 2, "playSub": 1}}}, map[string]uint64{"play:Sub": 2}), + Entry("audrey player", insights.Data{Library: insightsLibrary{ActivePlayers: map[string]int64{"eu.callcc.audrey": 4}}}, map[string]uint64{"audrey": 4}), + Entry("discard DSubCC player", insights.Data{Library: insightsLibrary{ActivePlayers: map[string]int64{"DSubCC": 5}}}, map[string]uint64{}), + Entry("bonob player", insights.Data{Library: insightsLibrary{ActivePlayers: map[string]int64{"bonob": 6, "bonob+ogg": 4}}}, map[string]uint64{"bonob": 6}), + Entry("Airsonic Refix player", insights.Data{Library: insightsLibrary{ActivePlayers: map[string]int64{"http://airsonic.netlify.app": 7}}}, map[string]uint64{"Airsonic Refix": 7}), + Entry("Airsonic Refix player (HTTPS)", insights.Data{Library: insightsLibrary{ActivePlayers: map[string]int64{"https://airsonic.netlify.app": 7}}}, map[string]uint64{"Airsonic Refix": 7}), + Entry("Multiple players", insights.Data{Library: insightsLibrary{ActivePlayers: map[string]int64{"Feishin": 1, "NavidromeUI_1.0": 2, "playSub_1.0": 3, "eu.callcc.audrey": 4, "DSubCC": 5, "bonob": 6, "bonob+ogg": 4, "http://airsonic.netlify.app": 7}}}, + map[string]uint64{"Feishin": 1, "NavidromeUI": 2, "play:Sub": 3, "audrey": 4, "bonob": 6, "Airsonic Refix": 7}), + ) +}) + +type insightsOS struct { + Type string `json:"type"` + Distro string `json:"distro,omitempty"` + Version string `json:"version,omitempty"` + Containerized bool `json:"containerized"` + Arch string `json:"arch"` + NumCPU int `json:"numCPU"` +} + +type insightsLibrary struct { + Tracks int64 `json:"tracks"` + Albums int64 `json:"albums"` + Artists int64 `json:"artists"` + Playlists int64 `json:"playlists"` + Shares int64 `json:"shares"` + Radios int64 `json:"radios"` + ActiveUsers int64 `json:"activeUsers"` + ActivePlayers map[string]int64 `json:"activePlayers,omitempty"` +} diff --git a/tasks.go b/tasks.go index 3f4df61..33201aa 100644 --- a/tasks.go +++ b/tasks.go @@ -16,7 +16,10 @@ func cleanup(_ context.Context, db *sql.DB) func() { func summarize(_ context.Context, db *sql.DB) func() { return func() { - log.Print("Summarizing data") - _ = summarizeData(db, time.Now()) + log.Print("Summarizing data for the last week") + now := time.Now().Truncate(24 * time.Hour).UTC() + for d := 0; d < 10; d++ { + _ = summarizeData(db, now.Add(-time.Duration(d)*24*time.Hour)) + } } } From 5ea542c42e81742b99487aab1c26f9392b443466 Mon Sep 17 00:00:00 2001 From: Deluan Date: Fri, 27 Dec 2024 11:12:03 -0500 Subject: [PATCH 4/9] Add std deviation --- summary.go | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/summary.go b/summary.go index 5263acd..99cd08f 100644 --- a/summary.go +++ b/summary.go @@ -6,6 +6,7 @@ import ( "fmt" "iter" "log" + "math" "regexp" "strings" "time" @@ -16,12 +17,14 @@ import ( ) type Summary struct { - Versions map[string]uint64 `json:"versions,omitempty"` - OS map[string]uint64 `json:"OS,omitempty"` - PlayerTypes map[string]uint64 `json:"playerTypes,omitempty"` - Players map[string]uint64 `json:"players,omitempty"` - Users map[string]uint64 `json:"users,omitempty"` - Tracks map[string]uint64 `json:"tracks,omitempty"` + Versions map[string]uint64 `json:"versions,omitempty"` + OS map[string]uint64 `json:"OS,omitempty"` + PlayerTypes map[string]uint64 `json:"playerTypes,omitempty"` + Players map[string]uint64 `json:"players,omitempty"` + Users map[string]uint64 `json:"users,omitempty"` + Tracks map[string]uint64 `json:"tracks,omitempty"` + LibSizeAverage int64 `json:"libSizeAverage,omitempty"` + LibSizeStdDev float64 `json:"libSizeStdDev,omitempty"` } func summarizeData(db *sql.DB, date time.Time) error { @@ -38,6 +41,9 @@ func summarizeData(db *sql.DB, date time.Time) error { Users: make(map[string]uint64), Tracks: make(map[string]uint64), } + var numInstances int64 + var sumTracks int64 + var sumTracksSquared int64 for data := range rows { // Summarize data here summary.Versions[mapVersion(data)]++ @@ -46,6 +52,17 @@ func summarizeData(db *sql.DB, date time.Time) error { totalPlayers := mapPlayerTypes(data, summary.PlayerTypes) summary.Players[fmt.Sprintf("%d", totalPlayers)]++ mapToBins(data.Library.Tracks, trackBins, summary.Tracks) + if data.Library.Tracks > 0 { + sumTracks += data.Library.Tracks + sumTracksSquared += data.Library.Tracks * data.Library.Tracks + numInstances++ + } + } + if numInstances > 0 { + summary.LibSizeAverage = sumTracks / numInstances + mean := float64(sumTracks) / float64(numInstances) + variance := float64(sumTracksSquared)/float64(numInstances) - mean*mean + summary.LibSizeStdDev = math.Sqrt(variance) } // Save summary to database err = saveSummary(db, summary, date) @@ -93,7 +110,7 @@ func mapOS(data insights.Data) string { var playersTypes = map[*regexp.Regexp]string{ regexp.MustCompile("NavidromeUI.*"): "NavidromeUI", regexp.MustCompile("supersonic"): "Supersonic", - regexp.MustCompile("feishin_"): "", // Discard (old version) + regexp.MustCompile("feishin"): "", // Discard (old version reporting multiple times) regexp.MustCompile("audioling"): "Audioling", regexp.MustCompile("playSub.*"): "play:Sub", regexp.MustCompile("eu.callcc.audrey"): "audrey", @@ -102,6 +119,7 @@ var playersTypes = map[*regexp.Regexp]string{ regexp.MustCompile("https?://airsonic.*"): "Airsonic Refix", regexp.MustCompile("multi-scrobbler.*"): "Multi-Scrobbler", regexp.MustCompile("SubMusic.*"): "SubMusic", + regexp.MustCompile("(?i)(hiby|_hiby_)"): "HiBy", } func mapPlayerTypes(data insights.Data, players map[string]uint64) int64 { From 1e116ccbcf7cb249b4b3d1c0dba0797585c0ecaf Mon Sep 17 00:00:00 2001 From: Deluan Date: Fri, 27 Dec 2024 12:14:00 -0500 Subject: [PATCH 5/9] Add fs type counts --- summary.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/summary.go b/summary.go index 99cd08f..1217889 100644 --- a/summary.go +++ b/summary.go @@ -23,6 +23,8 @@ type Summary struct { Players map[string]uint64 `json:"players,omitempty"` Users map[string]uint64 `json:"users,omitempty"` Tracks map[string]uint64 `json:"tracks,omitempty"` + MusicFS map[string]uint64 `json:"musicFS,omitempty"` + DataFS map[string]uint64 `json:"dataFS,omitempty"` LibSizeAverage int64 `json:"libSizeAverage,omitempty"` LibSizeStdDev float64 `json:"libSizeStdDev,omitempty"` } @@ -40,6 +42,8 @@ func summarizeData(db *sql.DB, date time.Time) error { Players: make(map[string]uint64), Users: make(map[string]uint64), Tracks: make(map[string]uint64), + MusicFS: make(map[string]uint64), + DataFS: make(map[string]uint64), } var numInstances int64 var sumTracks int64 @@ -49,6 +53,8 @@ func summarizeData(db *sql.DB, date time.Time) error { summary.Versions[mapVersion(data)]++ summary.OS[mapOS(data)]++ summary.Users[fmt.Sprintf("%d", data.Library.ActiveUsers)]++ + summary.MusicFS[mapFS(data.FS.Music)]++ + summary.DataFS[mapFS(data.FS.Data)]++ totalPlayers := mapPlayerTypes(data, summary.PlayerTypes) summary.Players[fmt.Sprintf("%d", totalPlayers)]++ mapToBins(data.Library.Tracks, trackBins, summary.Tracks) @@ -144,6 +150,26 @@ func mapPlayerTypes(data insights.Data, players map[string]uint64) int64 { return total } +var fsMappings = map[string]string{ + "unknown(0x2011bab0)": "exfat", + "unknown(0x7366746e)": "ntfs", + "unknown(0xc36400)": "ceph", + "unknown(0xf15f)": "ecryptfs", + "unknown(0xff534d42)": "cifs", + "unknown(0x786f4256)": "vboxsf", + "unknown(0xf2f52010)": "f2fs", +} + +func mapFS(fs *insights.FSInfo) string { + if fs == nil { + return "unknown" + } + if t, ok := fsMappings[fs.Type]; ok { + return t + } + return fs.Type +} + func selectData(db *sql.DB, date time.Time) (iter.Seq[insights.Data], error) { query := ` SELECT i1.id, i1.time, i1.data From fa6e6fd50a87a896fb35dadaba392e2a12347839 Mon Sep 17 00:00:00 2001 From: Deluan Date: Fri, 27 Dec 2024 12:14:28 -0500 Subject: [PATCH 6/9] Extend summary period calculation --- tasks.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks.go b/tasks.go index 33201aa..6c392a3 100644 --- a/tasks.go +++ b/tasks.go @@ -18,7 +18,7 @@ func summarize(_ context.Context, db *sql.DB) func() { return func() { log.Print("Summarizing data for the last week") now := time.Now().Truncate(24 * time.Hour).UTC() - for d := 0; d < 10; d++ { + for d := 0; d < 15; d++ { _ = summarizeData(db, now.Add(-time.Duration(d)*24*time.Hour)) } } From 11406c4e07dc8dafb7dcd654c7b9e2f8dea2fb21 Mon Sep 17 00:00:00 2001 From: Deluan Date: Thu, 9 Jan 2025 19:44:07 -0500 Subject: [PATCH 7/9] Fine tune summary --- main.go | 5 +++-- summary.go | 34 ++++++++++++++++++++++++++-------- summary_test.go | 11 +++++++++++ tasks.go | 8 +++++--- 4 files changed, 45 insertions(+), 13 deletions(-) diff --git a/main.go b/main.go index 562ee49..cb745d6 100644 --- a/main.go +++ b/main.go @@ -16,7 +16,8 @@ import ( func startTasks(ctx context.Context, db *sql.DB) error { c := cron.New() - _, err := c.AddFunc("1 * * * *", summarize(ctx, db)) + // Run summarize every two hours + _, err := c.AddFunc("0 */2 * * *", summarize(ctx, db)) if err != nil { return err } @@ -39,7 +40,7 @@ func main() { log.Fatal(err) } - summarize(ctx, db)() + go summarize(ctx, db)() r := chi.NewRouter() r.Use(middleware.RealIP) diff --git a/summary.go b/summary.go index 1217889..ee11e95 100644 --- a/summary.go +++ b/summary.go @@ -17,8 +17,11 @@ import ( ) type Summary struct { + NumInstances int64 `json:"numInstances,omitempty"` + NumActiveUsers int64 `json:"numActiveUsers,omitempty"` Versions map[string]uint64 `json:"versions,omitempty"` - OS map[string]uint64 `json:"OS,omitempty"` + OS map[string]uint64 `json:"os,omitempty"` + Distros map[string]uint64 `json:"distros,omitempty"` PlayerTypes map[string]uint64 `json:"playerTypes,omitempty"` Players map[string]uint64 `json:"players,omitempty"` Users map[string]uint64 `json:"users,omitempty"` @@ -38,6 +41,7 @@ func summarizeData(db *sql.DB, date time.Time) error { summary := Summary{ Versions: make(map[string]uint64), OS: make(map[string]uint64), + Distros: make(map[string]uint64), PlayerTypes: make(map[string]uint64), Players: make(map[string]uint64), Users: make(map[string]uint64), @@ -50,8 +54,13 @@ func summarizeData(db *sql.DB, date time.Time) error { var sumTracksSquared int64 for data := range rows { // Summarize data here + summary.NumInstances++ + summary.NumActiveUsers += data.Library.ActiveUsers summary.Versions[mapVersion(data)]++ summary.OS[mapOS(data)]++ + if data.OS.Type == "linux" && !data.OS.Containerized { + summary.Distros[data.OS.Distro]++ + } summary.Users[fmt.Sprintf("%d", data.Library.ActiveUsers)]++ summary.MusicFS[mapFS(data.FS.Music)]++ summary.DataFS[mapFS(data.FS.Data)]++ @@ -64,12 +73,15 @@ func summarizeData(db *sql.DB, date time.Time) error { numInstances++ } } - if numInstances > 0 { - summary.LibSizeAverage = sumTracks / numInstances - mean := float64(sumTracks) / float64(numInstances) - variance := float64(sumTracksSquared)/float64(numInstances) - mean*mean - summary.LibSizeStdDev = math.Sqrt(variance) + if numInstances == 0 { + log.Printf("No data to summarize for %s", date.Format("2006-01-02")) + return nil } + summary.LibSizeAverage = sumTracks / numInstances + mean := float64(sumTracks) / float64(numInstances) + variance := float64(sumTracksSquared)/float64(numInstances) - mean*mean + summary.LibSizeStdDev = math.Sqrt(variance) + // Save summary to database err = saveSummary(db, summary, date) if err != nil { @@ -79,7 +91,12 @@ func summarizeData(db *sql.DB, date time.Time) error { return err } -func mapVersion(data insights.Data) string { return data.Version } +// Match the first 8 characters of a git sha +var versionRegex = regexp.MustCompile(`\(([0-9a-fA-F]{8})[0-9a-fA-F]*\)`) + +func mapVersion(data insights.Data) string { + return versionRegex.ReplaceAllString(data.Version, "($1)") +} var trackBins = []int64{0, 1, 100, 500, 1000, 5000, 10000, 20000, 50000, 100000, 500000, 1000000} @@ -118,6 +135,7 @@ var playersTypes = map[*regexp.Regexp]string{ regexp.MustCompile("supersonic"): "Supersonic", regexp.MustCompile("feishin"): "", // Discard (old version reporting multiple times) regexp.MustCompile("audioling"): "Audioling", + regexp.MustCompile("^AginMusic.*"): "AginMusic", regexp.MustCompile("playSub.*"): "play:Sub", regexp.MustCompile("eu.callcc.audrey"): "audrey", regexp.MustCompile("DSubCC"): "", // Discard (chromecast) @@ -167,7 +185,7 @@ func mapFS(fs *insights.FSInfo) string { if t, ok := fsMappings[fs.Type]; ok { return t } - return fs.Type + return strings.ToLower(fs.Type) } func selectData(db *sql.DB, date time.Time) (iter.Seq[insights.Data], error) { diff --git a/summary_test.go b/summary_test.go index a9ae16e..70be2d3 100644 --- a/summary_test.go +++ b/summary_test.go @@ -58,6 +58,17 @@ var _ = Describe("Summary", func() { }) }) + DescribeTable("mapVersion", + func(expected string, data insights.Data) { + Expect(mapVersion(data)).To(Equal(expected)) + }, + Entry("should map version", "0.54.2 (0b184893)", insights.Data{Version: "0.54.2 (0b184893)"}), + Entry("should map version with long hash", "0.54.2 (0b184893)", insights.Data{Version: "0.54.2 (0b184893278620bb421a85c8b47df36900cd4df7)"}), + Entry("should map version with no hash", "dev", insights.Data{Version: "dev"}), + Entry("should map version with other values", "0.54.3 (source_archive)", insights.Data{Version: "0.54.3 (source_archive)"}), + Entry("should map any version with a hash", "0.54.3-SNAPSHOT (734eb30a)", insights.Data{Version: "0.54.3-SNAPSHOT (734eb30a)"}), + ) + DescribeTable("mapOS", func(expected string, data insights.Data) { Expect(mapOS(data)).To(Equal(expected)) diff --git a/tasks.go b/tasks.go index 6c392a3..2013761 100644 --- a/tasks.go +++ b/tasks.go @@ -16,10 +16,12 @@ func cleanup(_ context.Context, db *sql.DB) func() { func summarize(_ context.Context, db *sql.DB) func() { return func() { - log.Print("Summarizing data for the last week") + log.Print("Summarizing data") now := time.Now().Truncate(24 * time.Hour).UTC() - for d := 0; d < 15; d++ { - _ = summarizeData(db, now.Add(-time.Duration(d)*24*time.Hour)) + for d := 0; d < 45; d++ { + date := now.Add(-time.Duration(d) * 24 * time.Hour) + log.Print("Summarizing data for ", date.Format("2006-01-02")) + _ = summarizeData(db, date) } } } From b8abe96d96057135c7435c21409400687d5d5f35 Mon Sep 17 00:00:00 2001 From: Deluan Date: Fri, 10 Jan 2025 11:44:59 -0500 Subject: [PATCH 8/9] Add AVSub mapping --- summary.go | 1 + 1 file changed, 1 insertion(+) diff --git a/summary.go b/summary.go index ee11e95..aeca363 100644 --- a/summary.go +++ b/summary.go @@ -144,6 +144,7 @@ var playersTypes = map[*regexp.Regexp]string{ regexp.MustCompile("multi-scrobbler.*"): "Multi-Scrobbler", regexp.MustCompile("SubMusic.*"): "SubMusic", regexp.MustCompile("(?i)(hiby|_hiby_)"): "HiBy", + regexp.MustCompile("microSub"): "AVSub", } func mapPlayerTypes(data insights.Data, players map[string]uint64) int64 { From af1fd8b85cad517f629d94eefabdccc4e2088bdd Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 14 Jan 2025 13:07:30 -0500 Subject: [PATCH 9/9] give me a bit more time to figure out the summarization --- db.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db.go b/db.go index 304e95e..50104ed 100644 --- a/db.go +++ b/db.go @@ -76,7 +76,7 @@ func saveSummary(db *sql.DB, summary Summary, t time.Time) error { func purgeOldEntries(db *sql.DB) error { // Delete entries older than 30 days query := `DELETE FROM insights WHERE time < ?` - cnt, err := db.Exec(query, time.Now().Add(-30*24*time.Hour)) + cnt, err := db.Exec(query, time.Now().Add(-90*24*time.Hour)) if err != nil { return err }