From 7c338c00ea091ee1f8e3abc1e9c9c45a8dd74c3a Mon Sep 17 00:00:00 2001 From: iandyh Date: Fri, 12 Jan 2024 14:10:02 +0900 Subject: [PATCH 1/5] usage logic --- shibuya/api/main.go | 3 + shibuya/api/usage.go | 36 ++++++++ shibuya/model/collection.go | 10 +++ shibuya/model/common.go | 9 ++ shibuya/model/project.go | 40 +++++++++ shibuya/model/usage.go | 160 ++++++++++++++++++++++++++++++++++++ 6 files changed, 258 insertions(+) create mode 100644 shibuya/api/usage.go create mode 100644 shibuya/model/usage.go diff --git a/shibuya/api/main.go b/shibuya/api/main.go index ec96691d..a7524cac 100644 --- a/shibuya/api/main.go +++ b/shibuya/api/main.go @@ -849,6 +849,9 @@ func (s *ShibuyaAPI) InitRoutes() Routes { &Route{"files", "GET", "/api/files/:kind/:id/:name", s.fileDownloadHandler}, + &Route{"usage_summary", "GET", "/api/usage/summary", s.usageSummaryHandler}, + &Route{"usage_summary_by_sid", "GET", "/api/usage/summary_sid", s.usageSummaryHandlerBySid}, + &Route{"admin_collections", "GET", "/api/admin/collections", s.collectionAdminGetHandler}, } } diff --git a/shibuya/api/usage.go b/shibuya/api/usage.go new file mode 100644 index 00000000..64e6ae14 --- /dev/null +++ b/shibuya/api/usage.go @@ -0,0 +1,36 @@ +package api + +import ( + "net/http" + + log "github.com/sirupsen/logrus" + + "github.com/julienschmidt/httprouter" + "github.com/rakutentech/shibuya/shibuya/model" +) + +func (s *ShibuyaAPI) usageSummaryHandler(w http.ResponseWriter, req *http.Request, params httprouter.Params) { + qs := req.URL.Query() + st := qs.Get("started_time") + et := qs.Get("end_time") + summary, err := model.GetUsageSummary(st, et) + if err != nil { + log.Println(err) + s.handleErrors(w, err) + return + } + s.jsonise(w, http.StatusOK, summary) +} + +func (s *ShibuyaAPI) usageSummaryHandlerBySid(w http.ResponseWriter, req *http.Request, params httprouter.Params) { + qs := req.URL.Query() + st := qs.Get("started_time") + et := qs.Get("end_time") + sid := qs.Get("sid") + history, err := model.GetUsageSummaryBySid(sid, st, et) + if err != nil { + s.handleErrors(w, err) + return + } + s.jsonise(w, http.StatusOK, history) +} diff --git a/shibuya/model/collection.go b/shibuya/model/collection.go index e1e2bbca..14ca823f 100644 --- a/shibuya/model/collection.go +++ b/shibuya/model/collection.go @@ -34,6 +34,16 @@ type Collection struct { CSVSplit bool `json:"csv_split"` } +type CollectionLaunchHistory struct { + Context string `json:"context"` + CollectionID int64 `json:"collection_id"` + Owner string `json:"owner"` + Vu int `json:"vu"` + StartedTime time.Time `json:"started_time"` + EndTime time.Time `json:"end_time"` + BillingHours float64 `json:"billing_hours"` +} + func CreateCollection(name string, projectID int64) (int64, error) { DBC := config.SC.DBC q, err := DBC.Prepare("insert collection set name=?,project_id=?") diff --git a/shibuya/model/common.go b/shibuya/model/common.go index 20a9e1e9..b88fdbeb 100644 --- a/shibuya/model/common.go +++ b/shibuya/model/common.go @@ -3,3 +3,12 @@ package model const ( MySQLFormat = "2006-01-02 15:04:05" ) + +func inArray(s []string, item string) bool { + for _, m := range s { + if m == item { + return true + } + } + return false +} diff --git a/shibuya/model/project.go b/shibuya/model/project.go index 3486a870..9194a26c 100644 --- a/shibuya/model/project.go +++ b/shibuya/model/project.go @@ -161,3 +161,43 @@ func (p *Project) GetPlans() ([]*Plan, error) { } return r, nil } + +func GetProjectsBySid(sid string) ([]*Project, error) { + db := config.SC.DBC + r := []*Project{} + query := fmt.Sprintf("select id, name, owner, sid, created_time from project where sid=%s", sid) + q, err := db.Prepare(query) + if err != nil { + return r, err + } + defer q.Close() + rows, err := q.Query() + if err != nil { + return r, err + } + defer rows.Close() + for rows.Next() { + p := new(Project) + rows.Scan(&p.ID, &p.Name, &p.Owner, &p.SID, &p.CreatedTime) + r = append(r, p) + } + err = rows.Err() + if err != nil { + return r, err + } + return r, nil +} + +func (p *Project) UpdateSid(sid string) error { + db := config.SC.DBC + q, err := db.Prepare("update project set sid=? where id=?") + if err != nil { + return err + } + defer q.Close() + + if _, err := q.Exec(sid, p.ID); err != nil { + return err + } + return nil +} diff --git a/shibuya/model/usage.go b/shibuya/model/usage.go new file mode 100644 index 00000000..d7c78d15 --- /dev/null +++ b/shibuya/model/usage.go @@ -0,0 +1,160 @@ +package model + +import ( + "math" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/rakutentech/shibuya/shibuya/config" +) + +// vuh per context +// example map: +// gcp: 10, aws: 20 +type UnitUsage struct { + TotalVUH map[string]float64 `json:"total_vuh"` +} + +type TotalUsageSummary struct { + UnitUsage + VUHByOnwer map[string]map[string]float64 `json:"vuh_by_owner"` + Contacts map[string][]string `json:"contacts"` +} + +type OwnerUsageSummary struct { + UnitUsage + History []*CollectionLaunchHistory `json:"launch_history"` +} + +func GetHistory(startedTime, endTime string) ([]*CollectionLaunchHistory, error) { + db := config.SC.DBC + q, err := db.Prepare("select collection_id, context, owner, vu, started_time, end_time from collection_launch_history2 where started_time > ? and end_time < ?") + if err != nil { + return nil, err + } + rs, err := q.Query(startedTime, endTime) + defer rs.Close() + + history := []*CollectionLaunchHistory{} + for rs.Next() { + lh := new(CollectionLaunchHistory) + rs.Scan(&lh.CollectionID, &lh.Context, &lh.Owner, &lh.Vu, &lh.StartedTime, &lh.EndTime) + history = append(history, lh) + } + return history, nil +} + +func calBillingHours(startedTime, endTime time.Time) float64 { + duration := endTime.Sub(startedTime) + billingHours := math.Ceil(duration.Hours()) + return billingHours +} + +func calVUH(billingHours, vu float64) float64 { + return billingHours * vu +} + +func makeCollectionsToProjects(history []*CollectionLaunchHistory) map[int64]Project { + collectionsToProjects := make(map[int64]Project) + for _, h := range history { + cid := h.CollectionID + if _, ok := collectionsToProjects[cid]; ok { + continue + } + c, err := GetCollection(cid) + if err != nil { + continue + } + p, err := GetProject(c.ProjectID) + if err != nil { + continue + } + collectionsToProjects[cid] = *p + } + return collectionsToProjects +} + +func GetUsageSummary(startedTime, endTime string) (*TotalUsageSummary, error) { + history, err := GetHistory(startedTime, endTime) + if err != nil { + return nil, err + } + uu := UnitUsage{ + TotalVUH: make(map[string]float64), + } + s := &TotalUsageSummary{ + UnitUsage: uu, + VUHByOnwer: make(map[string]map[string]float64), + Contacts: make(map[string][]string), + } + collectionsToProjects := makeCollectionsToProjects(history) + for _, p := range collectionsToProjects { + sid := p.SID + if sid == "" { + sid = "unknwon" + } + contacts := s.Contacts[sid] + if !inArray(contacts, p.Owner) { + contacts = append(contacts, p.Owner) + s.Contacts[sid] = contacts + } + } + for _, h := range history { + totalVUH := uu.TotalVUH + vhByOwner := s.VUHByOnwer + project, ok := collectionsToProjects[h.CollectionID] + // the project has been deleted so we cannot find the project + // TODO we should directly use the sid in the history + if !ok { + continue + } + sid := "unknown" + if project.SID != "" { + sid = project.SID + } + // if users run 0.1 hours, we should bill them based on 1 hour. + billingHours := calBillingHours(h.StartedTime, h.EndTime) + vuh := calVUH(billingHours, float64(h.Vu)) + totalVUH[h.Context] += vuh + if m, ok := vhByOwner[sid]; !ok { + vhByOwner[sid] = make(map[string]float64) + vhByOwner[sid][h.Context] += vuh + } else { + m[h.Context] += vuh + } + } + return s, nil +} + +func GetUsageSummaryBySid(sid, startedTime, endTime string) (*OwnerUsageSummary, error) { + log.Printf("fetch history for %s", sid) + history, err := GetHistory(startedTime, endTime) + if err != nil { + return nil, err + } + collectionsToProjects := makeCollectionsToProjects(history) + uu := UnitUsage{ + TotalVUH: make(map[string]float64), + } + sidHistory := []*CollectionLaunchHistory{} + s := &OwnerUsageSummary{ + UnitUsage: uu, + } + for _, h := range history { + p, ok := collectionsToProjects[h.CollectionID] + if !ok { + continue + } + if p.SID != sid { + continue + } + sidHistory = append(sidHistory, h) + billingHours := calBillingHours(h.StartedTime, h.EndTime) + vuh := calVUH(billingHours, float64(h.Vu)) + uu.TotalVUH[h.Context] += vuh + h.BillingHours = billingHours + } + s.History = sidHistory + return s, nil +} From bfadc74025ffb1483dcc28fc840f00bda4ad0ec8 Mon Sep 17 00:00:00 2001 From: iandyh Date: Mon, 15 Jan 2024 14:14:43 +0900 Subject: [PATCH 2/5] billed by sid instead of by project owner mailing list --- shibuya/controller/main.go | 6 ++--- shibuya/model/usage.go | 50 ++++++++++++++++++++++++++++---------- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/shibuya/controller/main.go b/shibuya/controller/main.go index 015bbb66..0802dd0b 100644 --- a/shibuya/controller/main.go +++ b/shibuya/controller/main.go @@ -195,11 +195,11 @@ func (c *Controller) DeployCollection(collection *model.Collection) error { return err } } - owner := "" + sid := "" if project, err := model.GetProject(collection.ProjectID); err == nil { - owner = project.Owner + sid = project.SID } - if err := collection.NewLaunchEntry(owner, config.SC.Context, int64(enginesCount), nodesCount, int64(vu)); err != nil { + if err := collection.NewLaunchEntry(sid, config.SC.Context, int64(enginesCount), nodesCount, int64(vu)); err != nil { return err } err = utils.Retry(func() error { diff --git a/shibuya/model/usage.go b/shibuya/model/usage.go index d7c78d15..1b8007d1 100644 --- a/shibuya/model/usage.go +++ b/shibuya/model/usage.go @@ -2,6 +2,7 @@ package model import ( "math" + "strconv" "time" log "github.com/sirupsen/logrus" @@ -75,6 +76,13 @@ func makeCollectionsToProjects(history []*CollectionLaunchHistory) map[int64]Pro return collectionsToProjects } +func findOwner(owner string, project Project) string { + if _, err := strconv.ParseInt(owner, 10, 32); err != nil { + return project.SID + } + return owner +} + func GetUsageSummary(startedTime, endTime string) (*TotalUsageSummary, error) { history, err := GetHistory(startedTime, endTime) if err != nil { @@ -103,16 +111,24 @@ func GetUsageSummary(startedTime, endTime string) (*TotalUsageSummary, error) { for _, h := range history { totalVUH := uu.TotalVUH vhByOwner := s.VUHByOnwer - project, ok := collectionsToProjects[h.CollectionID] - // the project has been deleted so we cannot find the project - // TODO we should directly use the sid in the history - if !ok { - continue - } - sid := "unknown" - if project.SID != "" { - sid = project.SID + owner := h.Owner + sid := owner + _, err := strconv.ParseInt(owner, 10, 32) + + // the sid is using email. This could happen during transition period + // and we need to fetch the sid from project + if err != nil { + project, ok := collectionsToProjects[h.CollectionID] + // the project has been deleted so we cannot find the project + if !ok { + continue + } + sid = "unknown" + if project.SID != "" { + sid = project.SID + } } + // if users run 0.1 hours, we should bill them based on 1 hour. billingHours := calBillingHours(h.StartedTime, h.EndTime) vuh := calVUH(billingHours, float64(h.Vu)) @@ -142,11 +158,19 @@ func GetUsageSummaryBySid(sid, startedTime, endTime string) (*OwnerUsageSummary, UnitUsage: uu, } for _, h := range history { - p, ok := collectionsToProjects[h.CollectionID] - if !ok { - continue + owner := h.Owner + _, err := strconv.ParseInt(h.Owner, 10, 32) + if err != nil { + p, ok := collectionsToProjects[h.CollectionID] + if !ok { + continue + } + if p.SID != sid { + continue + } + owner = p.SID } - if p.SID != sid { + if owner != sid { continue } sidHistory = append(sidHistory, h) From b6a423e0fce8fc3325918f2174da21adf05f8028 Mon Sep 17 00:00:00 2001 From: iandyh Date: Mon, 15 Jan 2024 14:23:37 +0900 Subject: [PATCH 3/5] add some comments --- shibuya/model/usage.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shibuya/model/usage.go b/shibuya/model/usage.go index 1b8007d1..8a36cc7e 100644 --- a/shibuya/model/usage.go +++ b/shibuya/model/usage.go @@ -145,6 +145,8 @@ func GetUsageSummary(startedTime, endTime string) (*TotalUsageSummary, error) { func GetUsageSummaryBySid(sid, startedTime, endTime string) (*OwnerUsageSummary, error) { log.Printf("fetch history for %s", sid) + // we could implement something like get history by sid but this is not being indexed atm + // it will remain as a TODO in the future history, err := GetHistory(startedTime, endTime) if err != nil { return nil, err From d807e7e5217efd61aedca1136a3b76e3e83138d8 Mon Sep 17 00:00:00 2001 From: iandyh Date: Fri, 2 Feb 2024 15:27:59 +0900 Subject: [PATCH 4/5] remove unused code --- shibuya/model/project.go | 40 ---------------------------------------- 1 file changed, 40 deletions(-) diff --git a/shibuya/model/project.go b/shibuya/model/project.go index 9194a26c..3486a870 100644 --- a/shibuya/model/project.go +++ b/shibuya/model/project.go @@ -161,43 +161,3 @@ func (p *Project) GetPlans() ([]*Plan, error) { } return r, nil } - -func GetProjectsBySid(sid string) ([]*Project, error) { - db := config.SC.DBC - r := []*Project{} - query := fmt.Sprintf("select id, name, owner, sid, created_time from project where sid=%s", sid) - q, err := db.Prepare(query) - if err != nil { - return r, err - } - defer q.Close() - rows, err := q.Query() - if err != nil { - return r, err - } - defer rows.Close() - for rows.Next() { - p := new(Project) - rows.Scan(&p.ID, &p.Name, &p.Owner, &p.SID, &p.CreatedTime) - r = append(r, p) - } - err = rows.Err() - if err != nil { - return r, err - } - return r, nil -} - -func (p *Project) UpdateSid(sid string) error { - db := config.SC.DBC - q, err := db.Prepare("update project set sid=? where id=?") - if err != nil { - return err - } - defer q.Close() - - if _, err := q.Exec(sid, p.ID); err != nil { - return err - } - return nil -} From f9249564d1f2f98d7dfdf1753dadb168e347363a Mon Sep 17 00:00:00 2001 From: iandyh Date: Tue, 6 Feb 2024 09:33:48 +0900 Subject: [PATCH 5/5] polish the comment --- shibuya/model/usage.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/shibuya/model/usage.go b/shibuya/model/usage.go index 8a36cc7e..ade77206 100644 --- a/shibuya/model/usage.go +++ b/shibuya/model/usage.go @@ -113,6 +113,12 @@ func GetUsageSummary(startedTime, endTime string) (*TotalUsageSummary, error) { vhByOwner := s.VUHByOnwer owner := h.Owner sid := owner + + // we also change the ownership from ML to SID while keeping the old entries as they were. + // When we run the monthly billing, we could see mixed of ML and SID after first release, + // however, after a month or so, we should see all the entries to be billed by SID. + // During this transition period of time, we will parse the entry first and if it's still billed by + // ML, we will try to get the SID from its belonging SID. _, err := strconv.ParseInt(owner, 10, 32) // the sid is using email. This could happen during transition period @@ -129,7 +135,8 @@ func GetUsageSummary(startedTime, endTime string) (*TotalUsageSummary, error) { } } - // if users run 0.1 hours, we should bill them based on 1 hour. + // if users run less than 1 hour, we should bill them by 1 hour. + // 1 hour is the minimum charging unit. billingHours := calBillingHours(h.StartedTime, h.EndTime) vuh := calVUH(billingHours, float64(h.Vu)) totalVUH[h.Context] += vuh