From 5dd406d3eb88cc2457edaf8092fc5cb2fe3e4e3a Mon Sep 17 00:00:00 2001 From: ecrupper Date: Wed, 28 Feb 2024 10:50:39 -0600 Subject: [PATCH 1/4] feat: add types, fix errors, add conditional status report --- api/build/create.go | 2 +- api/build/restart.go | 2 +- api/step/plan.go | 17 +++++++++-------- api/step/update.go | 12 +++++++----- api/webhook/post.go | 2 +- cmd/vela-server/schedule.go | 2 +- go.mod | 2 ++ scm/github/github.go | 21 ++++++++++++++++----- scm/github/repo.go | 12 +++++++++--- 9 files changed, 47 insertions(+), 25 deletions(-) diff --git a/api/build/create.go b/api/build/create.go index 991946c88..88fa52908 100644 --- a/api/build/create.go +++ b/api/build/create.go @@ -320,7 +320,7 @@ func CreateBuild(c *gin.Context) { input.SetPipelineID(pipeline.GetID()) // create the objects from the pipeline in the database - err = PlanBuild(ctx, database.FromContext(c), p, input, r) + err = PlanBuild(ctx, database.FromContext(c), scm.FromContext(c), p, input, r) if err != nil { util.HandleError(c, http.StatusInternalServerError, err) diff --git a/api/build/restart.go b/api/build/restart.go index 1296d654a..782ab313a 100644 --- a/api/build/restart.go +++ b/api/build/restart.go @@ -319,7 +319,7 @@ func RestartBuild(c *gin.Context) { b.SetPipelineID(pipeline.GetID()) // create the objects from the pipeline in the database - err = PlanBuild(ctx, database.FromContext(c), p, b, r) + err = PlanBuild(ctx, database.FromContext(c), scm.FromContext(c), p, b, r) if err != nil { util.HandleError(c, http.StatusInternalServerError, err) diff --git a/api/step/plan.go b/api/step/plan.go index 6d77c8b5e..f31f76db9 100644 --- a/api/step/plan.go +++ b/api/step/plan.go @@ -38,7 +38,7 @@ func PlanSteps(ctx context.Context, database database.Interface, scm scm.Service // iterate through all pipeline steps for _, step := range p.Steps { - s, err := planStep(ctx, database, b, step, "") + s, err := planStep(ctx, database, scm, b, r, step, "") if err != nil { return steps, err } @@ -61,14 +61,15 @@ func planStep(ctx context.Context, database database.Interface, scm scm.Service, s.SetStatus(constants.StatusPending) s.SetCreated(time.Now().UTC().Unix()) - id, err := scm.CreateChecks(ctx, r, b.GetCommit(), s.GetName()) - if err != nil { - // TODO: make this error more meaningful - return nil, err - } + if c.ReportStatus { + id, err := scm.CreateChecks(ctx, r, b.GetCommit(), s.GetName()) + if err != nil { + // TODO: make this error more meaningful + return nil, err + } - // TODO: have to store the check ID somewhere - s.SetCheckID(id) + s.SetCheckID(id) + } // send API call to create the step s, err := database.CreateStep(ctx, s) diff --git a/api/step/update.go b/api/step/update.go index cdd3f2bd9..a0fdedd77 100644 --- a/api/step/update.go +++ b/api/step/update.go @@ -157,13 +157,15 @@ func UpdateStep(c *gin.Context) { return } - err = scm.FromContext(c).UpdateChecks(ctx, r, s, b.GetCommit()) - if err != nil { - retErr := fmt.Errorf("unable to set step check %s: %w", entry, err) + if s.GetCheckID() != 0 { + err = scm.FromContext(c).UpdateChecks(ctx, r, s, b.GetCommit()) + if err != nil { + retErr := fmt.Errorf("unable to set step check %s: %w", entry, err) - util.HandleError(c, http.StatusInternalServerError, retErr) + util.HandleError(c, http.StatusInternalServerError, retErr) - return + return + } } c.JSON(http.StatusOK, s) diff --git a/api/webhook/post.go b/api/webhook/post.go index 038d53fe7..1ca54c40f 100644 --- a/api/webhook/post.go +++ b/api/webhook/post.go @@ -589,7 +589,7 @@ func PostWebhook(c *gin.Context) { // using the same Number and thus create a constraint // conflict; consider deleting the partially created // build object in the database - err = build.PlanBuild(ctx, database.FromContext(c), p, b, repo) + err = build.PlanBuild(ctx, database.FromContext(c), scm.FromContext(c), p, b, repo) if err != nil { retErr := fmt.Errorf("%s: %w", baseErr, err) diff --git a/cmd/vela-server/schedule.go b/cmd/vela-server/schedule.go index 2036306c4..f31de1948 100644 --- a/cmd/vela-server/schedule.go +++ b/cmd/vela-server/schedule.go @@ -356,7 +356,7 @@ func processSchedule(ctx context.Context, s *library.Schedule, compiler compiler // using the same Number and thus create a constraint // conflict; consider deleting the partially created // build object in the database - err = build.PlanBuild(ctx, database, p, b, r) + err = build.PlanBuild(ctx, database, scm, p, b, r) if err != nil { // check if the retry limit has been exceeded if i < retryLimit-1 { diff --git a/go.mod b/go.mod index 4eba76d21..db34dcd0b 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/go-vela/server go 1.21 +replace github.com/go-vela/types => ../types + require ( github.com/Bose/minisentinel v0.0.0-20200130220412-917c5a9223bb github.com/DATA-DOG/go-sqlmock v1.5.2 diff --git a/scm/github/github.go b/scm/github/github.go index ac17af4f4..3bd38795d 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -186,17 +186,28 @@ func (c *client) newClientToken(token string) *github.Client { } // helper function to return the GitHub App token. -func (c *client) newGithubAppToken(r *library.Repo) *github.Client { +func (c *client) newGithubAppToken(r *library.Repo) (*github.Client, error) { // create a github client based off the existing GitHub App configuration client, err := github.NewClient(&http.Client{Transport: c.AppsTransport}).WithEnterpriseURLs(c.config.API, c.config.API) if err != nil { - panic(err) + return nil, err + } + + // if repo has an install ID, use it to create an installation token + if r.GetInstallID() != 0 { + // create installation token for the repo + t, _, err := client.Apps.CreateInstallationToken(context.Background(), r.GetInstallID(), &github.InstallationTokenOptions{}) + if err != nil { + panic(err) + } + + return c.newClientToken(t.GetToken()), nil } // list all installations (a.k.a. orgs) where the GitHub App is installed installations, _, err := client.Apps.ListInstallations(context.Background(), &github.ListOptions{}) if err != nil { - panic(err) + return nil, err } var id int64 @@ -210,7 +221,7 @@ func (c *client) newGithubAppToken(r *library.Repo) *github.Client { // failsafe in case the repo does not belong to an org where the GitHub App is installed if id == 0 { - panic(err) + return nil, err } // create installation token for the repo @@ -219,5 +230,5 @@ func (c *client) newGithubAppToken(r *library.Repo) *github.Client { panic(err) } - return c.newClientToken(t.GetToken()) + return c.newClientToken(t.GetToken()), nil } diff --git a/scm/github/repo.go b/scm/github/repo.go index 48e837f6f..eb7ea693d 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -589,7 +589,10 @@ func (c *client) GetBranch(ctx context.Context, u *library.User, r *library.Repo // CreateChecks defines a function that does stuff... func (c *client) CreateChecks(ctx context.Context, r *library.Repo, commit, step string) (int64, error) { // create client from GitHub App - client := c.newGithubAppToken(r) + client, err := c.newGithubAppToken(r) + if err != nil { + return 0, err + } opts := github.CreateCheckRunOptions{ Name: fmt.Sprintf("vela-%s-%s", commit, step), @@ -607,7 +610,10 @@ func (c *client) CreateChecks(ctx context.Context, r *library.Repo, commit, step // UpdateChecks defines a function that does stuff... func (c *client) UpdateChecks(ctx context.Context, r *library.Repo, s *library.Step, commit string) error { // create client from GitHub App - client := c.newGithubAppToken(r) + client, err := c.newGithubAppToken(r) + if err != nil { + return err + } var ( conclusion string @@ -650,7 +656,7 @@ func (c *client) UpdateChecks(ctx context.Context, r *library.Repo, s *library.S Status: github.String(status), } - _, _, err := client.Checks.UpdateCheckRun(ctx, r.GetOrg(), r.GetName(), s.GetCheckID(), opts) + _, _, err = client.Checks.UpdateCheckRun(ctx, r.GetOrg(), r.GetName(), s.GetCheckID(), opts) if err != nil { return err } From 823b44ea14e68b1d7477283ce40a087cb1e789d7 Mon Sep 17 00:00:00 2001 From: ecrupper Date: Wed, 28 Feb 2024 12:15:28 -0600 Subject: [PATCH 2/4] add db funcs and source private key from base64 string --- api/step/plan.go | 2 +- api/step/update.go | 2 +- database/repo/table.go | 2 ++ database/step/table.go | 3 +++ scm/github/github.go | 20 ++++++++++++++++++-- scm/github/repo.go | 8 ++++---- scm/service.go | 4 ++-- 7 files changed, 31 insertions(+), 10 deletions(-) diff --git a/api/step/plan.go b/api/step/plan.go index f31f76db9..e1c446c44 100644 --- a/api/step/plan.go +++ b/api/step/plan.go @@ -62,7 +62,7 @@ func planStep(ctx context.Context, database database.Interface, scm scm.Service, s.SetCreated(time.Now().UTC().Unix()) if c.ReportStatus { - id, err := scm.CreateChecks(ctx, r, b.GetCommit(), s.GetName()) + id, err := scm.CreateChecks(ctx, r, b.GetCommit(), s.GetName(), b.GetEvent()) if err != nil { // TODO: make this error more meaningful return nil, err diff --git a/api/step/update.go b/api/step/update.go index a0fdedd77..d1e31d4a0 100644 --- a/api/step/update.go +++ b/api/step/update.go @@ -158,7 +158,7 @@ func UpdateStep(c *gin.Context) { } if s.GetCheckID() != 0 { - err = scm.FromContext(c).UpdateChecks(ctx, r, s, b.GetCommit()) + err = scm.FromContext(c).UpdateChecks(ctx, r, s, b.GetCommit(), b.GetEvent()) if err != nil { retErr := fmt.Errorf("unable to set step check %s: %w", entry, err) diff --git a/database/repo/table.go b/database/repo/table.go index e0f26a6c3..c28b6412f 100644 --- a/database/repo/table.go +++ b/database/repo/table.go @@ -40,6 +40,7 @@ repos ( pipeline_type TEXT, previous_name VARCHAR(100), approve_build VARCHAR(20), + install_id INTEGER, UNIQUE(full_name) ); ` @@ -75,6 +76,7 @@ repos ( pipeline_type TEXT, previous_name TEXT, approve_build TEXT, + install_id INTEGER, UNIQUE(full_name) ); ` diff --git a/database/step/table.go b/database/step/table.go index 8cdde1b6e..c322d67a2 100644 --- a/database/step/table.go +++ b/database/step/table.go @@ -4,6 +4,7 @@ package step import ( "context" + "github.com/go-vela/types/constants" ) @@ -29,6 +30,7 @@ steps ( host VARCHAR(250), runtime VARCHAR(250), distribution VARCHAR(250), + check_id INTEGER, UNIQUE(build_id, number) ); ` @@ -54,6 +56,7 @@ steps ( host TEXT, runtime TEXT, distribution TEXT, + check_id INTEGER, UNIQUE(build_id, number) ); ` diff --git a/scm/github/github.go b/scm/github/github.go index 3bd38795d..782e924c8 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -4,6 +4,9 @@ package github import ( "context" + "crypto/x509" + "encoding/base64" + "encoding/pem" "fmt" "net/http" "net/url" @@ -120,11 +123,24 @@ func New(opts ...ClientOpt) (*client, error) { if c.config.GithubAppID != 0 && len(c.config.GithubAppPrivateKey) > 0 { c.Logger.Infof("sourcing private key from path: %s", c.config.GithubAppPrivateKey) - transport, err := ghinstallation.NewAppsTransportKeyFromFile(http.DefaultTransport, c.config.GithubAppID, c.config.GithubAppPrivateKey) + + decodedPEM, err := base64.StdEncoding.DecodeString(c.config.GithubAppPrivateKey) if err != nil { - return nil, err + return nil, fmt.Errorf("error decoding base64: %w", err) + } + + block, _ := pem.Decode(decodedPEM) + if block == nil { + return nil, fmt.Errorf("failed to parse PEM block containing the key") } + privateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + return nil, fmt.Errorf("failed to parse RSA private key: %w", err) + } + + transport := ghinstallation.NewAppsTransportFromPrivateKey(http.DefaultTransport, c.config.GithubAppID, privateKey) + transport.BaseURL = c.config.API c.AppsTransport = transport } diff --git a/scm/github/repo.go b/scm/github/repo.go index eb7ea693d..27031b114 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -587,7 +587,7 @@ func (c *client) GetBranch(ctx context.Context, u *library.User, r *library.Repo } // CreateChecks defines a function that does stuff... -func (c *client) CreateChecks(ctx context.Context, r *library.Repo, commit, step string) (int64, error) { +func (c *client) CreateChecks(ctx context.Context, r *library.Repo, commit, step, event string) (int64, error) { // create client from GitHub App client, err := c.newGithubAppToken(r) if err != nil { @@ -595,7 +595,7 @@ func (c *client) CreateChecks(ctx context.Context, r *library.Repo, commit, step } opts := github.CreateCheckRunOptions{ - Name: fmt.Sprintf("vela-%s-%s", commit, step), + Name: fmt.Sprintf("vela-%s-%s", event, step), HeadSHA: commit, } @@ -608,7 +608,7 @@ func (c *client) CreateChecks(ctx context.Context, r *library.Repo, commit, step } // UpdateChecks defines a function that does stuff... -func (c *client) UpdateChecks(ctx context.Context, r *library.Repo, s *library.Step, commit string) error { +func (c *client) UpdateChecks(ctx context.Context, r *library.Repo, s *library.Step, commit, event string) error { // create client from GitHub App client, err := c.newGithubAppToken(r) if err != nil { @@ -651,7 +651,7 @@ func (c *client) UpdateChecks(ctx context.Context, r *library.Repo, s *library.S } opts := github.UpdateCheckRunOptions{ - Name: fmt.Sprintf("vela-%s-%s", commit, s.GetName()), + Name: fmt.Sprintf("vela-%s-%s", event, s.GetName()), Conclusion: github.String(conclusion), Status: github.String(status), } diff --git a/scm/service.go b/scm/service.go index daf3ff7ed..df4672f3e 100644 --- a/scm/service.go +++ b/scm/service.go @@ -133,8 +133,8 @@ type Service interface { GetHTMLURL(context.Context, *library.User, string, string, string, string) (string, error) // TODO: add comments - CreateChecks(context.Context, *library.Repo, string, string) (int64, error) - UpdateChecks(context.Context, *library.Repo, *library.Step, string) error + CreateChecks(context.Context, *library.Repo, string, string, string) (int64, error) + UpdateChecks(context.Context, *library.Repo, *library.Step, string, string) error // Webhook SCM Interface Functions From 5dd7d638a4b49eecbd39a8a243457ac3d17b34f3 Mon Sep 17 00:00:00 2001 From: ecrupper Date: Fri, 1 Mar 2024 08:51:52 -0600 Subject: [PATCH 3/4] include status report --- api/step/update.go | 2 ++ go.sum | 2 -- scm/github/repo.go | 28 ++++++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/api/step/update.go b/api/step/update.go index d1e31d4a0..49bb82e9b 100644 --- a/api/step/update.go +++ b/api/step/update.go @@ -158,6 +158,8 @@ func UpdateStep(c *gin.Context) { } if s.GetCheckID() != 0 { + s.SetReport(input.GetReport()) + err = scm.FromContext(c).UpdateChecks(ctx, r, s, b.GetCommit(), b.GetEvent()) if err != nil { retErr := fmt.Errorf("unable to set step check %s: %w", entry, err) diff --git a/go.sum b/go.sum index 38a2dc84a..1980be9bd 100644 --- a/go.sum +++ b/go.sum @@ -88,8 +88,6 @@ github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw= github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= -github.com/go-vela/types v0.23.1 h1:st4BeDcYVyaaFqblU1YroztNvmYLBgmfZpWq0En0Sg0= -github.com/go-vela/types v0.23.1/go.mod h1:AAqgxIw1aRBgPkE/5juGuiwh/JZuOtL8fcPaEkjFWwQ= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= diff --git a/scm/github/repo.go b/scm/github/repo.go index 27031b114..7415196f3 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -650,10 +650,38 @@ func (c *client) UpdateChecks(ctx context.Context, r *library.Repo, s *library.S status = "completed" } + var annotations []*github.CheckRunAnnotation + + for _, reportAnnotation := range s.GetReport().GetAnnotations() { + annotation := &github.CheckRunAnnotation{ + Path: github.String(reportAnnotation.GetPath()), + StartLine: github.Int(reportAnnotation.GetStartLine()), + EndLine: github.Int(reportAnnotation.GetEndLine()), + StartColumn: github.Int(reportAnnotation.GetStartColumn()), + EndColumn: github.Int(reportAnnotation.GetEndColumn()), + AnnotationLevel: github.String(reportAnnotation.GetAnnotationLevel()), + Message: github.String(reportAnnotation.GetMessage()), + Title: github.String(reportAnnotation.GetTitle()), + RawDetails: github.String(reportAnnotation.GetRawDetails()), + } + + annotations = append(annotations, annotation) + } + + output := &github.CheckRunOutput{ + Title: github.String(s.GetReport().GetTitle()), + Summary: github.String(s.GetReport().GetSummary()), + Text: github.String(s.GetReport().GetText()), + AnnotationsCount: github.Int(s.GetReport().GetAnnotationsCount()), + AnnotationsURL: github.String(s.GetReport().GetAnnotationsURL()), + Annotations: annotations, + } + opts := github.UpdateCheckRunOptions{ Name: fmt.Sprintf("vela-%s-%s", event, s.GetName()), Conclusion: github.String(conclusion), Status: github.String(status), + Output: output, } _, _, err = client.Checks.UpdateCheckRun(ctx, r.GetOrg(), r.GetName(), s.GetCheckID(), opts) From 83d28ac1da3a3709b9d92626cef823bba15c6267 Mon Sep 17 00:00:00 2001 From: ecrupper Date: Mon, 24 Jun 2024 15:12:14 -0500 Subject: [PATCH 4/4] fix table syntax --- database/step/table.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/database/step/table.go b/database/step/table.go index 445e7ac50..900734de4 100644 --- a/database/step/table.go +++ b/database/step/table.go @@ -30,11 +30,8 @@ steps ( host VARCHAR(250), runtime VARCHAR(250), distribution VARCHAR(250), -<<<<<<< HEAD check_id INTEGER, -======= report_as VARCHAR(250), ->>>>>>> hackathon/github_app UNIQUE(build_id, number) ); ` @@ -60,11 +57,8 @@ steps ( host TEXT, runtime TEXT, distribution TEXT, -<<<<<<< HEAD check_id INTEGER, -======= report_as TEXT, ->>>>>>> hackathon/github_app UNIQUE(build_id, number) ); `