From 4dc041d750a81dc13e9ca839e5503269a6204c4f Mon Sep 17 00:00:00 2001 From: Deluan Date: Thu, 26 Dec 2024 23:45:49 -0500 Subject: [PATCH] Fix summary --- main.go | 4 +- summary.go | 71 ++++++++++++++++++++++-------- summary_test.go | 114 ++++++++++++++++++++++++++++++++++++++++++++++++ tasks.go | 7 ++- 4 files changed, 174 insertions(+), 22 deletions(-) create mode 100644 summary_test.go diff --git a/main.go b/main.go index 257d28a..562ee49 100644 --- a/main.go +++ b/main.go @@ -16,7 +16,7 @@ import ( func startTasks(ctx context.Context, db *sql.DB) error { c := cron.New() - _, err := c.AddFunc("1 0 * * *", summarize(ctx, db)) + _, err := c.AddFunc("1 * * * *", summarize(ctx, db)) if err != nil { return err } @@ -39,6 +39,8 @@ func main() { log.Fatal(err) } + summarize(ctx, db)() + r := chi.NewRouter() r.Use(middleware.RealIP) r.Use(middleware.Logger) diff --git a/summary.go b/summary.go index 8896dc1..5263acd 100644 --- a/summary.go +++ b/summary.go @@ -6,6 +6,7 @@ import ( "fmt" "iter" "log" + "regexp" "strings" "time" @@ -15,11 +16,12 @@ import ( ) type Summary struct { - Versions map[string]uint64 - OS map[string]uint64 - Players map[string]uint64 - Users map[string]uint64 - Tracks map[string]uint64 + Versions map[string]uint64 `json:"versions,omitempty"` + OS map[string]uint64 `json:"OS,omitempty"` + PlayerTypes map[string]uint64 `json:"playerTypes,omitempty"` + Players map[string]uint64 `json:"players,omitempty"` + Users map[string]uint64 `json:"users,omitempty"` + Tracks map[string]uint64 `json:"tracks,omitempty"` } func summarizeData(db *sql.DB, date time.Time) error { @@ -29,18 +31,20 @@ func summarizeData(db *sql.DB, date time.Time) error { return err } summary := Summary{ - Versions: make(map[string]uint64), - OS: make(map[string]uint64), - Players: make(map[string]uint64), - Users: make(map[string]uint64), - Tracks: make(map[string]uint64), + Versions: make(map[string]uint64), + OS: make(map[string]uint64), + PlayerTypes: make(map[string]uint64), + Players: make(map[string]uint64), + Users: make(map[string]uint64), + Tracks: make(map[string]uint64), } for data := range rows { // Summarize data here summary.Versions[mapVersion(data)]++ summary.OS[mapOS(data)]++ - mapPlayers(data, summary.Players) - mapToBins(data.Library.ActiveUsers, userBins, summary.Users) + summary.Users[fmt.Sprintf("%d", data.Library.ActiveUsers)]++ + totalPlayers := mapPlayerTypes(data, summary.PlayerTypes) + summary.Players[fmt.Sprintf("%d", totalPlayers)]++ mapToBins(data.Library.Tracks, trackBins, summary.Tracks) } // Save summary to database @@ -55,7 +59,6 @@ func summarizeData(db *sql.DB, date time.Time) error { func mapVersion(data insights.Data) string { return data.Version } var trackBins = []int64{0, 1, 100, 500, 1000, 5000, 10000, 20000, 50000, 100000, 500000, 1000000} -var userBins = []int64{0, 1, 5, 10, 20, 50, 100, 200, 500, 1000} func mapToBins(count int64, bins []int64, counters map[string]uint64) { for i := range bins { @@ -80,17 +83,47 @@ func mapOS(data insights.Data) string { } return "Linux" default: - s := strings.Replace(data.OS.Type, "bsd", "BSD", -1) - return caser.String(s) + s := caser.String(data.OS.Type) + return strings.Replace(s, "bsd", "BSD", -1) } }() return os + " - " + data.OS.Arch } -func mapPlayers(data insights.Data, players map[string]uint64) { +var playersTypes = map[*regexp.Regexp]string{ + regexp.MustCompile("NavidromeUI.*"): "NavidromeUI", + regexp.MustCompile("supersonic"): "Supersonic", + regexp.MustCompile("feishin_"): "", // Discard (old version) + regexp.MustCompile("audioling"): "Audioling", + regexp.MustCompile("playSub.*"): "play:Sub", + regexp.MustCompile("eu.callcc.audrey"): "audrey", + regexp.MustCompile("DSubCC"): "", // Discard (chromecast) + regexp.MustCompile(`bonob\+.*`): "", // Discard (transcodings) + regexp.MustCompile("https?://airsonic.*"): "Airsonic Refix", + regexp.MustCompile("multi-scrobbler.*"): "Multi-Scrobbler", + regexp.MustCompile("SubMusic.*"): "SubMusic", +} + +func mapPlayerTypes(data insights.Data, players map[string]uint64) int64 { + seen := map[string]uint64{} for p, count := range data.Library.ActivePlayers { - players[p] = uint64(count) + for r, t := range playersTypes { + if r.MatchString(p) { + p = t + break + } + } + if p != "" { + v := seen[p] + seen[p] = max(v, uint64(count)) + } + } + var total int64 + for k, v := range seen { + total += int64(v) + players[k] += v } + return total } func selectData(db *sql.DB, date time.Time) (iter.Seq[insights.Data], error) { @@ -100,10 +133,10 @@ FROM insights i1 INNER JOIN ( SELECT id, MAX(time) as max_time FROM insights - WHERE time >= date(?, '-1 day') AND time < date(?) + WHERE time >= date(?) AND time < date(?, '+1 day') GROUP BY id ) i2 ON i1.id = i2.id AND i1.time = i2.max_time -WHERE i1.time >= date(?, '-1 day') AND time < date(?) +WHERE i1.time >= date(?) AND time < date(?, '+1 day') ORDER BY i1.id, i1.time DESC;` d := date.Format("2006-01-02") rows, err := db.Query(query, d, d, d, d) diff --git a/summary_test.go b/summary_test.go new file mode 100644 index 0000000..adabfe6 --- /dev/null +++ b/summary_test.go @@ -0,0 +1,114 @@ +package main + +import ( + "maps" + "slices" + "testing" + + "github.com/navidrome/navidrome/core/metrics/insights" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestSummary(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Insights Suite") +} + +var _ = Describe("Summary", func() { + Describe("mapToBins", func() { + var counters map[string]uint64 + var testBins = []int64{0, 1, 5, 10, 20, 50, 100, 200, 500, 1000} + + BeforeEach(func() { + counters = make(map[string]uint64) + }) + + It("should map count to the correct bin", func() { + mapToBins(0, testBins, counters) + Expect(counters["0"]).To(Equal(uint64(1))) + + mapToBins(1, testBins, counters) + Expect(counters["1"]).To(Equal(uint64(1))) + + mapToBins(10, testBins, counters) + Expect(counters["10"]).To(Equal(uint64(1))) + + mapToBins(101, testBins, counters) + Expect(counters["100"]).To(Equal(uint64(1))) + + mapToBins(1000, testBins, counters) + Expect(counters["1000"]).To(Equal(uint64(1))) + }) + + It("should map count to the highest bin if count exceeds all bins", func() { + mapToBins(2000, testBins, counters) + Expect(counters["1000"]).To(Equal(uint64(1))) + }) + + It("should increment the correct bin count", func() { + mapToBins(5, testBins, counters) + mapToBins(5, testBins, counters) + Expect(counters["5"]).To(Equal(uint64(2))) + }) + + It("should handle empty bins array", func() { + mapToBins(5, []int64{}, counters) + Expect(counters).To(BeEmpty()) + }) + }) + + DescribeTable("mapOS", + func(expected string, data insights.Data) { + Expect(mapOS(data)).To(Equal(expected)) + }, + Entry("should map darwin to macOS", "macOS - x86_64", insights.Data{OS: insightsOS{Type: "darwin", Arch: "x86_64"}}), + Entry("should map linux to Linux", "Linux - x86_64", insights.Data{OS: insightsOS{Type: "linux", Arch: "x86_64"}}), + Entry("should map containerized linux to Linux (containerized)", "Linux (containerized) - x86_64", insights.Data{OS: insightsOS{Type: "linux", Containerized: true, Arch: "x86_64"}}), + Entry("should map bsd to BSD", "FreeBSD - x86_64", insights.Data{OS: insightsOS{Type: "freebsd", Arch: "x86_64"}}), + Entry("should map unknown OS types", "Unknown - x86_64", insights.Data{OS: insightsOS{Type: "unknown", Arch: "x86_64"}}), + ) + DescribeTable("mapPlayerTypes", + func(data insights.Data, expected map[string]uint64) { + players := make(map[string]uint64) + c := mapPlayerTypes(data, players) + Expect(players).To(Equal(expected)) + values := slices.Collect(maps.Values(expected)) + var total uint64 + for _, v := range values { + total += v + } + Expect(c).To(Equal(int64(total))) + }, + Entry("Feishin player", insights.Data{Library: insightsLibrary{ActivePlayers: map[string]int64{"feishin_": 1, "Feishin": 1}}}, map[string]uint64{"Feishin": 1}), + Entry("NavidromeUI player", insights.Data{Library: insightsLibrary{ActivePlayers: map[string]int64{"NavidromeUI_1.0": 2}}}, map[string]uint64{"NavidromeUI": 2}), + FEntry("play:Sub player", insights.Data{Library: insightsLibrary{ActivePlayers: map[string]int64{"playSub_iPhone11": 2, "playSub": 1}}}, map[string]uint64{"play:Sub": 2}), + Entry("audrey player", insights.Data{Library: insightsLibrary{ActivePlayers: map[string]int64{"eu.callcc.audrey": 4}}}, map[string]uint64{"audrey": 4}), + Entry("discard DSubCC player", insights.Data{Library: insightsLibrary{ActivePlayers: map[string]int64{"DSubCC": 5}}}, map[string]uint64{}), + Entry("bonob player", insights.Data{Library: insightsLibrary{ActivePlayers: map[string]int64{"bonob": 6, "bonob+ogg": 4}}}, map[string]uint64{"bonob": 6}), + Entry("Airsonic Refix player", insights.Data{Library: insightsLibrary{ActivePlayers: map[string]int64{"http://airsonic.netlify.app": 7}}}, map[string]uint64{"Airsonic Refix": 7}), + Entry("Airsonic Refix player (HTTPS)", insights.Data{Library: insightsLibrary{ActivePlayers: map[string]int64{"https://airsonic.netlify.app": 7}}}, map[string]uint64{"Airsonic Refix": 7}), + Entry("Multiple players", insights.Data{Library: insightsLibrary{ActivePlayers: map[string]int64{"Feishin": 1, "NavidromeUI_1.0": 2, "playSub_1.0": 3, "eu.callcc.audrey": 4, "DSubCC": 5, "bonob": 6, "bonob+ogg": 4, "http://airsonic.netlify.app": 7}}}, + map[string]uint64{"Feishin": 1, "NavidromeUI": 2, "play:Sub": 3, "audrey": 4, "bonob": 6, "Airsonic Refix": 7}), + ) +}) + +type insightsOS struct { + Type string `json:"type"` + Distro string `json:"distro,omitempty"` + Version string `json:"version,omitempty"` + Containerized bool `json:"containerized"` + Arch string `json:"arch"` + NumCPU int `json:"numCPU"` +} + +type insightsLibrary struct { + Tracks int64 `json:"tracks"` + Albums int64 `json:"albums"` + Artists int64 `json:"artists"` + Playlists int64 `json:"playlists"` + Shares int64 `json:"shares"` + Radios int64 `json:"radios"` + ActiveUsers int64 `json:"activeUsers"` + ActivePlayers map[string]int64 `json:"activePlayers,omitempty"` +} diff --git a/tasks.go b/tasks.go index 3f4df61..33201aa 100644 --- a/tasks.go +++ b/tasks.go @@ -16,7 +16,10 @@ func cleanup(_ context.Context, db *sql.DB) func() { func summarize(_ context.Context, db *sql.DB) func() { return func() { - log.Print("Summarizing data") - _ = summarizeData(db, time.Now()) + log.Print("Summarizing data for the last week") + now := time.Now().Truncate(24 * time.Hour).UTC() + for d := 0; d < 10; d++ { + _ = summarizeData(db, now.Add(-time.Duration(d)*24*time.Hour)) + } } }