Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

enhance(pagination)!: remove total results calculation for performance improvement #1254

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

enhance(pagination)!: remove total results calculation for performance improvement #1254

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions api/build/graph.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -249,7 +249,7 @@ func GetBuildGraph(c *gin.Context) {
if len(p.Stages) > 0 || len(p.Steps) > 0 {
for page > 0 {
// retrieve build steps (per page) from the database
stepsPart, _, err := database.FromContext(c).ListStepsForBuild(ctx, b, nil, page, perPage)
stepsPart, err := database.FromContext(c).ListStepsForBuild(ctx, b, nil, page, perPage)
if err != nil {
retErr := fmt.Errorf("unable to retrieve steps for build %s: %w", entry, err)

Expand Down Expand Up @@ -287,7 +287,7 @@ func GetBuildGraph(c *gin.Context) {
if len(p.Services) > 0 {
for page > 0 {
// retrieve build services (per page) from the database
servicesPart, _, err := database.FromContext(c).ListServicesForBuild(ctx, b, nil, page, perPage)
servicesPart, err := database.FromContext(c).ListServicesForBuild(ctx, b, nil, page, perPage)
if err != nil {
retErr := fmt.Errorf("unable to retrieve services for build %s: %w", entry, err)

Expand Down
5 changes: 2 additions & 3 deletions api/build/list_org.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,6 @@ func ListBuildsForOrg(c *gin.Context) {
var (
filters = map[string]interface{}{}
b []*types.Build
t int64
)

// capture middleware values
Expand Down Expand Up @@ -199,7 +198,7 @@ func ListBuildsForOrg(c *gin.Context) {
}

// send API call to capture the list of builds for the org (and event type if passed in)
b, t, err = database.FromContext(c).ListBuildsForOrg(ctx, o, filters, page, perPage)
b, err = database.FromContext(c).ListBuildsForOrg(ctx, o, filters, page, perPage)
if err != nil {
retErr := fmt.Errorf("unable to list builds for org %s: %w", o, err)

Expand All @@ -212,7 +211,7 @@ func ListBuildsForOrg(c *gin.Context) {
pagination := api.Pagination{
Page: page,
PerPage: perPage,
Total: t,
Results: len(b),
}
// set pagination headers
pagination.SetHeaderLink(c)
Expand Down
5 changes: 2 additions & 3 deletions api/build/list_repo.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,6 @@ func ListBuildsForRepo(c *gin.Context) {
var (
filters = map[string]interface{}{}
b []*types.Build
t int64
)

// capture middleware values
Expand Down Expand Up @@ -237,7 +236,7 @@ func ListBuildsForRepo(c *gin.Context) {
return
}

b, t, err = database.FromContext(c).ListBuildsForRepo(ctx, r, filters, before, after, page, perPage)
b, err = database.FromContext(c).ListBuildsForRepo(ctx, r, filters, before, after, page, perPage)
if err != nil {
retErr := fmt.Errorf("unable to list builds for repo %s: %w", r.GetFullName(), err)

Expand All @@ -250,7 +249,7 @@ func ListBuildsForRepo(c *gin.Context) {
pagination := api.Pagination{
Page: page,
PerPage: perPage,
Total: t,
Results: len(b),
}
// set pagination headers
pagination.SetHeaderLink(c)
Expand Down
4 changes: 2 additions & 2 deletions api/build/update.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ func UpdateComponentStatuses(c *gin.Context, b *types.Build, status string) erro

for page > 0 {
// retrieve build steps (per page) from the database
stepsPart, _, err := database.FromContext(c).ListStepsForBuild(ctx, b, map[string]interface{}{}, page, perPage)
stepsPart, err := database.FromContext(c).ListStepsForBuild(ctx, b, map[string]interface{}{}, page, perPage)
if err != nil {
return err
}
Expand Down Expand Up @@ -234,7 +234,7 @@ func UpdateComponentStatuses(c *gin.Context, b *types.Build, status string) erro

for page > 0 {
// retrieve build services (per page) from the database
servicesPart, _, err := database.FromContext(c).ListServicesForBuild(ctx, b, map[string]interface{}{}, page, perPage)
servicesPart, err := database.FromContext(c).ListServicesForBuild(ctx, b, map[string]interface{}{}, page, perPage)
if err != nil {
return err
}
Expand Down
12 changes: 1 addition & 11 deletions api/deployment/list.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,16 +109,6 @@ func ListDeployments(c *gin.Context) {
// ensure per_page isn't above or below allowed values
perPage = max(1, min(100, perPage))

// send API call to capture the total number of deployments for the repo
t, err := database.FromContext(c).CountDeploymentsForRepo(c, r)
if err != nil {
retErr := fmt.Errorf("unable to get deployment count for %s: %w", r.GetFullName(), err)

util.HandleError(c, http.StatusInternalServerError, retErr)

return
}

// send API call to capture the list of deployments for the repo
d, err := database.FromContext(c).ListDeploymentsForRepo(c, r, page, perPage)
if err != nil {
Expand All @@ -133,7 +123,7 @@ func ListDeployments(c *gin.Context) {
pagination := api.Pagination{
Page: page,
PerPage: perPage,
Total: t,
Results: len(d),
}
// set pagination headers
pagination.SetHeaderLink(c)
Expand Down
4 changes: 2 additions & 2 deletions api/hook/list.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ func ListHooks(c *gin.Context) {
perPage = max(1, min(100, perPage))

// send API call to capture the list of steps for the build
h, t, err := database.FromContext(c).ListHooksForRepo(ctx, r, page, perPage)
h, err := database.FromContext(c).ListHooksForRepo(ctx, r, page, perPage)
if err != nil {
retErr := fmt.Errorf("unable to get hooks for repo %s: %w", r.GetFullName(), err)

Expand All @@ -125,7 +125,7 @@ func ListHooks(c *gin.Context) {
pagination := api.Pagination{
Page: page,
PerPage: perPage,
Total: t,
Results: len(h),
}
// set pagination headers
pagination.SetHeaderLink(c)
Expand Down
4 changes: 2 additions & 2 deletions api/log/list_build.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ func ListLogsForBuild(c *gin.Context) {
perPage = max(1, min(100, perPage))

// send API call to capture the list of logs for the build
bl, t, err := database.FromContext(c).ListLogsForBuild(ctx, b, page, perPage)
bl, err := database.FromContext(c).ListLogsForBuild(ctx, b, page, perPage)
if err != nil {
retErr := fmt.Errorf("unable to list logs for build %s: %w", entry, err)

Expand All @@ -125,7 +125,7 @@ func ListLogsForBuild(c *gin.Context) {
pagination := api.Pagination{
Page: page,
PerPage: perPage,
Total: t,
Results: len(bl),
}
// set pagination headers
pagination.SetHeaderLink(c)
Expand Down
21 changes: 4 additions & 17 deletions api/pagination.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ package api

import (
"fmt"
"math"
"net/http"
"strconv"
"strings"
Expand All @@ -16,7 +15,7 @@ import (
type Pagination struct {
PerPage int
Page int
Total int64
Results int
}

// HeaderLink will hold the information needed to form a link element in the header.
Expand All @@ -35,7 +34,6 @@ func (p *Pagination) SetHeaderLink(c *gin.Context) {

hl := HeaderLink{
"first": 1,
"last": p.TotalPages(),
"next": p.NextPage(),
"prev": p.PrevPage(),
}
Expand All @@ -55,7 +53,7 @@ func (p *Pagination) SetHeaderLink(c *gin.Context) {
}

// drop last, next on the last page
if p.Page == p.TotalPages() {
if p.Results < p.PerPage {
delete(hl, "last")
delete(hl, "next")
}
Expand All @@ -77,7 +75,6 @@ func (p *Pagination) SetHeaderLink(c *gin.Context) {
l = append(l, ls)
}

c.Header("X-Total-Count", strconv.FormatInt(p.Total, 10))
c.Header("Link", strings.Join(l, ", "))
}

Expand Down Expand Up @@ -106,22 +103,12 @@ func (p *Pagination) HasPrev() bool {

// HasNext will return true if there is a next page.
func (p *Pagination) HasNext() bool {
return p.Page < p.TotalPages()
return p.PerPage == p.Results
}

// HasPages returns true if there is need to deal with pagination.
func (p *Pagination) HasPages() bool {
return p.Total > int64(p.PerPage)
}

// TotalPages will return the total number of pages.
func (p *Pagination) TotalPages() int {
n := int(math.Ceil(float64(p.Total) / float64(p.PerPage)))
if n == 0 {
n = 1
}

return n
return !(p.Page == 1 && p.Results < p.PerPage)
}

// resolveScheme is a helper to determine the protocol scheme
Expand Down
4 changes: 2 additions & 2 deletions api/pipeline/list.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ func ListPipelines(c *gin.Context) {
//nolint:gomnd // ignore magic number
perPage = max(1, min(100, perPage))

p, t, err := database.FromContext(c).ListPipelinesForRepo(ctx, r, page, perPage)
p, err := database.FromContext(c).ListPipelinesForRepo(ctx, r, page, perPage)
if err != nil {
retErr := fmt.Errorf("unable to list pipelines for repo %s: %w", r.GetFullName(), err)

Expand All @@ -128,7 +128,7 @@ func ListPipelines(c *gin.Context) {
pagination := api.Pagination{
Page: page,
PerPage: perPage,
Total: t,
Results: len(p),
}
// set pagination headers
pagination.SetHeaderLink(c)
Expand Down
4 changes: 2 additions & 2 deletions api/repo/list.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ func ListRepos(c *gin.Context) {
}

// send API call to capture the list of repos for the user
r, t, err := database.FromContext(c).ListReposForUser(ctx, u, sortBy, filters, page, perPage)
r, err := database.FromContext(c).ListReposForUser(ctx, u, sortBy, filters, page, perPage)
if err != nil {
retErr := fmt.Errorf("unable to get repos for user %s: %w", u.GetName(), err)

Expand All @@ -121,7 +121,7 @@ func ListRepos(c *gin.Context) {
pagination := api.Pagination{
Page: page,
PerPage: perPage,
Total: t,
Results: len(r),
}
// set pagination headers
pagination.SetHeaderLink(c)
Expand Down
4 changes: 2 additions & 2 deletions api/repo/list_org.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ func ListReposForOrg(c *gin.Context) {
}

// send API call to capture the list of repos for the org
r, t, err := database.FromContext(c).ListReposForOrg(ctx, o, sortBy, filters, page, perPage)
r, err := database.FromContext(c).ListReposForOrg(ctx, o, sortBy, filters, page, perPage)
if err != nil {
retErr := fmt.Errorf("unable to get repos for org %s: %w", o, err)

Expand All @@ -158,7 +158,7 @@ func ListReposForOrg(c *gin.Context) {
pagination := api.Pagination{
Page: page,
PerPage: perPage,
Total: t,
Results: len(r),
}
// set pagination headers
pagination.SetHeaderLink(c)
Expand Down
4 changes: 2 additions & 2 deletions api/schedule/list.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ func ListSchedules(c *gin.Context) {
perPage = max(1, min(100, perPage))

// send API call to capture the list of schedules for the repo
s, t, err := database.FromContext(c).ListSchedulesForRepo(ctx, r, page, perPage)
s, err := database.FromContext(c).ListSchedulesForRepo(ctx, r, page, perPage)
if err != nil {
retErr := fmt.Errorf("unable to get schedules for repo %s: %w", r.GetFullName(), err)

Expand All @@ -124,7 +124,7 @@ func ListSchedules(c *gin.Context) {
pagination := api.Pagination{
Page: page,
PerPage: perPage,
Total: t,
Results: len(s),
}
// set pagination headers
pagination.SetHeaderLink(c)
Expand Down
2 changes: 1 addition & 1 deletion api/scm/sync_org.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ func SyncReposForOrg(c *gin.Context) {
page := 0
// capture all repos belonging to a certain org in database
for orgRepos := int64(0); orgRepos < t; orgRepos += 100 {
r, _, err := database.FromContext(c).ListReposForOrg(ctx, o, "name", map[string]interface{}{}, page, 100)
r, err := database.FromContext(c).ListReposForOrg(ctx, o, "name", map[string]interface{}{}, page, 100)
if err != nil {
retErr := fmt.Errorf("unable to get repo count for org %s: %w", o, err)

Expand Down
12 changes: 1 addition & 11 deletions api/secret/list.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,16 +162,6 @@ func ListSecrets(c *gin.Context) {
return
}

// send API call to capture the total number of secrets
total, err := secret.FromContext(c, e).Count(ctx, t, o, n, teams)
if err != nil {
retErr := fmt.Errorf("unable to get secret count for %s from %s service: %w", entry, e, err)

util.HandleError(c, http.StatusInternalServerError, retErr)

return
}

// ensure per_page isn't above or below allowed values
perPage = max(1, min(100, perPage))

Expand All @@ -189,7 +179,7 @@ func ListSecrets(c *gin.Context) {
pagination := api.Pagination{
Page: page,
PerPage: perPage,
Total: total,
Results: len(s),
}
// set pagination headers
pagination.SetHeaderLink(c)
Expand Down
4 changes: 2 additions & 2 deletions api/service/list.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ func ListServices(c *gin.Context) {
perPage = max(1, min(100, perPage))

// send API call to capture the list of services for the build
s, t, err := database.FromContext(c).ListServicesForBuild(ctx, b, map[string]interface{}{}, page, perPage)
s, err := database.FromContext(c).ListServicesForBuild(ctx, b, map[string]interface{}{}, page, perPage)
if err != nil {
retErr := fmt.Errorf("unable to get services for build %s: %w", entry, err)

Expand All @@ -133,7 +133,7 @@ func ListServices(c *gin.Context) {
pagination := api.Pagination{
Page: page,
PerPage: perPage,
Total: t,
Results: len(s),
}
// set pagination headers
pagination.SetHeaderLink(c)
Expand Down
4 changes: 2 additions & 2 deletions api/step/list.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ func ListSteps(c *gin.Context) {
perPage = max(1, min(100, perPage))

// send API call to capture the list of steps for the build
s, t, err := database.FromContext(c).ListStepsForBuild(ctx, b, map[string]interface{}{}, page, perPage)
s, err := database.FromContext(c).ListStepsForBuild(ctx, b, map[string]interface{}{}, page, perPage)
if err != nil {
retErr := fmt.Errorf("unable to list steps for build %s: %w", entry, err)

Expand All @@ -133,7 +133,7 @@ func ListSteps(c *gin.Context) {
pagination := api.Pagination{
Page: page,
PerPage: perPage,
Total: t,
Results: len(s),
}
// set pagination headers
pagination.SetHeaderLink(c)
Expand Down
2 changes: 1 addition & 1 deletion api/user/get_source.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ func GetSourceRepos(c *gin.Context) {

for page > 0 {
// send API call to capture the list of repos for the org
dbReposPart, _, err := database.FromContext(c).ListReposForOrg(ctx, org, "name", filters, page, 100)
dbReposPart, err := database.FromContext(c).ListReposForOrg(ctx, org, "name", filters, page, 100)
if err != nil {
retErr := fmt.Errorf("unable to get repos for org %s: %w", org, err)

Expand Down
4 changes: 2 additions & 2 deletions api/user/list.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ func ListUsers(c *gin.Context) {
perPage = max(1, min(100, perPage))

// send API call to capture the list of users
users, t, err := database.FromContext(c).ListLiteUsers(ctx, page, perPage)
users, err := database.FromContext(c).ListLiteUsers(ctx, page, perPage)
if err != nil {
retErr := fmt.Errorf("unable to get users: %w", err)

Expand All @@ -108,7 +108,7 @@ func ListUsers(c *gin.Context) {
pagination := api.Pagination{
Page: page,
PerPage: perPage,
Total: t,
Results: len(users),
}
// set pagination headers
pagination.SetHeaderLink(c)
Expand Down
Loading
Loading