Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

enhance: supply parent context to scm funcs #1169

Merged
merged 6 commits into from
Aug 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 18 additions & 18 deletions api/build/compile_publish.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ type CompileAndPublishConfig struct {
//
//nolint:funlen,gocyclo // ignore function length due to comments, error handling, and general complexity of function
func CompileAndPublish(
c context.Context,
ctx context.Context,
cfg CompileAndPublishConfig,
database database.Interface,
scm scm.Service,
Expand All @@ -66,7 +66,7 @@ func CompileAndPublish(
baseErr := cfg.BaseErr

// confirm current repo owner has at least write access to repo (needed for status update later)
_, err := scm.RepoAccess(c, u.GetName(), u.GetToken(), r.GetOrg(), r.GetName())
_, err := scm.RepoAccess(ctx, u.GetName(), u.GetToken(), r.GetOrg(), r.GetName())
if err != nil {
retErr := fmt.Errorf("unable to publish build to queue: repository owner %s no longer has write access to repository %s", u.GetName(), r.GetFullName())

Expand All @@ -87,7 +87,7 @@ func CompileAndPublish(
// if the event is issue_comment and the issue is a pull request,
// call SCM for more data not provided in webhook payload
if strings.EqualFold(cfg.Source, "webhook") && strings.EqualFold(b.GetEvent(), constants.EventComment) {
commit, branch, baseref, headref, err := scm.GetPullRequest(c, r, prNum)
commit, branch, baseref, headref, err := scm.GetPullRequest(ctx, r, prNum)
if err != nil {
retErr := fmt.Errorf("%s: failed to get pull request info for %s: %w", baseErr, r.GetFullName(), err)

Expand All @@ -103,7 +103,7 @@ func CompileAndPublish(
// if the source is from a schedule, fetch the commit sha from schedule branch (same as build branch at this moment)
if strings.EqualFold(cfg.Source, "schedule") {
// send API call to capture the commit sha for the branch
_, commit, err := scm.GetBranch(c, r, b.GetBranch())
_, commit, err := scm.GetBranch(ctx, r, b.GetBranch())
if err != nil {
retErr := fmt.Errorf("failed to get commit for repo %s on %s branch: %w", r.GetFullName(), r.GetBranch(), err)

Expand All @@ -119,7 +119,7 @@ func CompileAndPublish(
}

// send API call to capture the number of pending or running builds for the repo
builds, err := database.CountBuildsForRepo(c, r, filters)
builds, err := database.CountBuildsForRepo(ctx, r, filters)
if err != nil {
retErr := fmt.Errorf("%s: unable to get count of builds for repo %s", baseErr, r.GetFullName())

Expand Down Expand Up @@ -156,7 +156,7 @@ func CompileAndPublish(
!strings.EqualFold(b.GetEvent(), constants.EventPull) &&
!strings.EqualFold(b.GetEvent(), constants.EventDelete) {
// send API call to capture list of files changed for the commit
files, err = scm.Changeset(c, r, b.GetCommit())
files, err = scm.Changeset(ctx, r, b.GetCommit())
if err != nil {
retErr := fmt.Errorf("%s: failed to get changeset for %s: %w", baseErr, r.GetFullName(), err)

Expand All @@ -167,7 +167,7 @@ func CompileAndPublish(
// check if the build event is a pull_request
if strings.EqualFold(b.GetEvent(), constants.EventPull) && prNum > 0 {
// send API call to capture list of files changed for the pull request
files, err = scm.ChangesetPR(c, r, prNum)
files, err = scm.ChangesetPR(ctx, r, prNum)
if err != nil {
retErr := fmt.Errorf("%s: failed to get changeset for %s: %w", baseErr, r.GetFullName(), err)

Expand Down Expand Up @@ -202,10 +202,10 @@ func CompileAndPublish(
}

// send database call to attempt to capture the pipeline if we already processed it before
pipeline, err = database.GetPipelineForRepo(c, b.GetCommit(), r)
pipeline, err = database.GetPipelineForRepo(ctx, b.GetCommit(), r)
if err != nil { // assume the pipeline doesn't exist in the database yet
// send API call to capture the pipeline configuration file
pipelineFile, err = scm.ConfigBackoff(c, u, r, b.GetCommit())
pipelineFile, err = scm.ConfigBackoff(ctx, u, r, b.GetCommit())
if err != nil {
retErr := fmt.Errorf("%s: unable to get pipeline configuration for %s: %w", baseErr, r.GetFullName(), err)

Expand All @@ -216,7 +216,7 @@ func CompileAndPublish(
}

// send API call to capture repo for the counter (grabbing repo again to ensure counter is correct)
repo, err = database.GetRepoForOrg(c, r.GetOrg(), r.GetName())
repo, err = database.GetRepoForOrg(ctx, r.GetOrg(), r.GetName())
if err != nil {
retErr := fmt.Errorf("%s: unable to get repo %s: %w", baseErr, r.GetFullName(), err)

Expand Down Expand Up @@ -269,7 +269,7 @@ func CompileAndPublish(
WithRepo(repo).
WithUser(u).
WithLabels(cfg.Labels).
Compile(pipelineFile)
Compile(ctx, pipelineFile)
if err != nil {
// format the error message with extra information
err = fmt.Errorf("unable to compile pipeline configuration for %s: %w", repo.GetFullName(), err)
Expand All @@ -295,7 +295,7 @@ func CompileAndPublish(
b.SetStatus(constants.StatusSkipped)

// send API call to set the status on the commit
err = scm.Status(c, u, b, repo.GetOrg(), repo.GetName())
err = scm.Status(ctx, u, b, repo.GetOrg(), repo.GetName())
if err != nil {
logger.Errorf("unable to set commit status for %s/%d: %v", repo.GetFullName(), b.GetNumber(), err)
}
Expand All @@ -316,7 +316,7 @@ func CompileAndPublish(
pipeline.SetRef(b.GetRef())

// send API call to create the pipeline
pipeline, err = database.CreatePipeline(c, pipeline)
pipeline, err = database.CreatePipeline(ctx, pipeline)
if err != nil {
retErr := fmt.Errorf("%s: failed to create pipeline for %s: %w", baseErr, repo.GetFullName(), err)

Expand Down Expand Up @@ -348,7 +348,7 @@ func CompileAndPublish(
// using the same Number and thus create a constraint
// conflict; consider deleting the partially created
// build object in the database
err = PlanBuild(c, database, scm, p, b, repo)
err = PlanBuild(ctx, database, scm, p, b, repo)
if err != nil {
retErr := fmt.Errorf("%s: %w", baseErr, err)

Expand All @@ -373,7 +373,7 @@ func CompileAndPublish(
} // end of retry loop

// send API call to update repo for ensuring counter is incremented
repo, err = database.UpdateRepo(c, repo)
repo, err = database.UpdateRepo(ctx, repo)
if err != nil {
retErr := fmt.Errorf("%s: failed to update repo %s: %w", baseErr, repo.GetFullName(), err)

Expand Down Expand Up @@ -401,7 +401,7 @@ func CompileAndPublish(
}

// send API call to capture the triggered build
b, err = database.GetBuildForRepo(c, repo, b.GetNumber())
b, err = database.GetBuildForRepo(ctx, repo, b.GetNumber())
if err != nil {
retErr := fmt.Errorf("%s: failed to get new build %s/%d: %w", baseErr, repo.GetFullName(), b.GetNumber(), err)

Expand All @@ -414,7 +414,7 @@ func CompileAndPublish(
retErr := fmt.Errorf("unable to set route for build %d for %s: %w", b.GetNumber(), r.GetFullName(), err)

// error out the build
CleanBuild(c, database, b, nil, nil, retErr)
CleanBuild(ctx, database, b, nil, nil, retErr)

return nil, nil, http.StatusBadRequest, retErr
}
Expand All @@ -423,7 +423,7 @@ func CompileAndPublish(
b.SetHost(route)

// publish the pipeline.Build to the build_executables table to be requested by a worker
err = PublishBuildExecutable(c, database, p, b)
err = PublishBuildExecutable(ctx, database, p, b)
if err != nil {
retErr := fmt.Errorf("unable to publish build executable for %s/%d: %w", repo.GetFullName(), b.GetNumber(), err)

Expand Down
2 changes: 1 addition & 1 deletion api/build/graph.go
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@ func GetBuildGraph(c *gin.Context) {
WithMetadata(m).
WithRepo(r).
WithUser(u).
Compile(config)
Compile(ctx, config)
if err != nil {
// format the error message with extra information
err = fmt.Errorf("unable to compile pipeline configuration for %s: %w", r.GetFullName(), err)
Expand Down
3 changes: 2 additions & 1 deletion api/pipeline/compile.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ func CompilePipeline(c *gin.Context) {
p := pMiddleware.Retrieve(c)
r := repo.Retrieve(c)
u := user.Retrieve(c)
ctx := c.Request.Context()

entry := fmt.Sprintf("%s/%s", r.GetFullName(), p.GetCommit())

Expand All @@ -98,7 +99,7 @@ func CompilePipeline(c *gin.Context) {
ruleData := prepareRuleData(c)

// compile the pipeline
pipeline, _, err := compiler.CompileLite(p.GetData(), ruleData, true)
pipeline, _, err := compiler.CompileLite(ctx, p.GetData(), ruleData, true)
if err != nil {
retErr := fmt.Errorf("unable to compile pipeline %s: %w", entry, err)

Expand Down
3 changes: 2 additions & 1 deletion api/pipeline/expand.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ func ExpandPipeline(c *gin.Context) {
p := pipeline.Retrieve(c)
r := repo.Retrieve(c)
u := user.Retrieve(c)
ctx := c.Request.Context()

entry := fmt.Sprintf("%s/%s", r.GetFullName(), p.GetCommit())

Expand All @@ -98,7 +99,7 @@ func ExpandPipeline(c *gin.Context) {
ruleData := prepareRuleData(c)

// expand the templates in the pipeline
pipeline, _, err := compiler.CompileLite(p.GetData(), ruleData, false)
pipeline, _, err := compiler.CompileLite(ctx, p.GetData(), ruleData, false)
if err != nil {
retErr := fmt.Errorf("unable to expand pipeline %s: %w", entry, err)

Expand Down
2 changes: 1 addition & 1 deletion api/pipeline/template.go
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ func GetTemplates(c *gin.Context) {
templates[name] = template.ToLibrary()

// create a compiler registry client for parsing (no address or token needed for Parse)
registry, err := github.New("", "")
registry, err := github.New(ctx, "", "")
if err != nil {
util.HandleError(c, http.StatusBadRequest, fmt.Errorf("%s: unable to create compiler github client: %w", baseErr, err))

Expand Down
3 changes: 2 additions & 1 deletion api/pipeline/validate.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// SPDX-License-Identifier: Apache-2.0

package pipeline

Check failure on line 3 in api/pipeline/validate.go

View workflow job for this annotation

GitHub Actions / golangci

[golangci] api/pipeline/validate.go#L3

3-110 lines are duplicate of `api/pipeline/expand.go:4-112` (dupl)
Raw output
api/pipeline/validate.go:3: 3-110 lines are duplicate of `api/pipeline/expand.go:4-112` (dupl)
package pipeline

import (
	"fmt"
	"net/http"

	"github.com/gin-gonic/gin"
	"github.com/sirupsen/logrus"

	"github.com/go-vela/server/compiler"
	"github.com/go-vela/server/internal"
	"github.com/go-vela/server/router/middleware/pipeline"
	"github.com/go-vela/server/router/middleware/repo"
	"github.com/go-vela/server/router/middleware/user"
	"github.com/go-vela/server/util"
)

// swagger:operation POST /api/v1/pipelines/{org}/{repo}/{pipeline}/validate pipelines ValidatePipeline
//
// Get, expand and validate a pipeline
//
// ---
// produces:
// - application/yaml
// - application/json
// parameters:
// - in: path
//   name: org
//   description: Name of the organization
//   required: true
//   type: string
// - in: path
//   name: repo
//   description: Name of the repository
//   required: true
//   type: string
// - in: path
//   name: pipeline
//   description: Commit SHA for pipeline to retrieve
//   required: true
//   type: string
// - in: query
//   name: output
//   description: Output string for specifying output format
//   type: string
//   default: yaml
//   enum:
//   - json
//   - yaml
// security:
//   - ApiKeyAuth: []
// responses:
//   '200':
//     description: Successfully retrieved, expanded and validated the pipeline
//     schema:
//       type: string
//   '400':
//     description: Invalid request payload or path
//     schema:
//       "$ref": "#/definitions/Error"
//   '401':
//     description: Unauthorized
//     schema:
//       "$ref": "#/definitions/Error"
//   '404':
//     description: Not found
//     schema:
//       "$ref": "#/definitions/Error"
//   '500':
//     description: Unexpected server error
//     schema:
//       "$ref": "#/definitions/Error"

// ValidatePipeline represents the API handler to capture,
// expand and validate a pipeline configuration.
func ValidatePipeline(c *gin.Context) {
	// capture middleware values
	m := c.MustGet("metadata").(*internal.Metadata)
	l := c.MustGet("logger").(*logrus.Entry)
	p := pipeline.Retrieve(c)
	r := repo.Retrieve(c)
	u := user.Retrieve(c)
	ctx := c.Request.Context()

	entry := fmt.Sprintf("%s/%s", r.GetFullName(), p.GetCommit())

	l.Debugf("validating pipeline %s", entry)

	// ensure we use the expected pipeline type when compiling
	r.SetPipelineType(p.GetType())

	// create the compiler object
	compiler := compiler.FromContext(c).Duplicate().WithCommit(p.GetCommit()).WithMetadata(m).WithRepo(r).WithUser(u)

	ruleData := prepareRuleData(c)

	// validate the pipeline
	pipeline, _, err := compiler.CompileLite(ctx, p.GetData(), ruleData, false)
	if err != nil {
		retErr := fmt.Errorf("unable to validate pipeline %s: %w", entry, err)

		util.HandleError(c, http.StatusBadRequest, retErr)

		return
	}

	writeOutput(c, pipeline)
}

import (
"fmt"
Expand Down Expand Up @@ -75,13 +75,14 @@

// ValidatePipeline represents the API handler to capture,
// expand and validate a pipeline configuration.
func ValidatePipeline(c *gin.Context) {

Check failure on line 78 in api/pipeline/validate.go

View workflow job for this annotation

GitHub Actions / golangci

[golangci] api/pipeline/validate.go#L78

78-110 lines are duplicate of `api/pipeline/compile.go:80-112` (dupl)
Raw output
api/pipeline/validate.go:78: 78-110 lines are duplicate of `api/pipeline/compile.go:80-112` (dupl)
func ValidatePipeline(c *gin.Context) {
	// capture middleware values
	m := c.MustGet("metadata").(*internal.Metadata)
	l := c.MustGet("logger").(*logrus.Entry)
	p := pipeline.Retrieve(c)
	r := repo.Retrieve(c)
	u := user.Retrieve(c)
	ctx := c.Request.Context()

	entry := fmt.Sprintf("%s/%s", r.GetFullName(), p.GetCommit())

	l.Debugf("validating pipeline %s", entry)

	// ensure we use the expected pipeline type when compiling
	r.SetPipelineType(p.GetType())

	// create the compiler object
	compiler := compiler.FromContext(c).Duplicate().WithCommit(p.GetCommit()).WithMetadata(m).WithRepo(r).WithUser(u)

	ruleData := prepareRuleData(c)

	// validate the pipeline
	pipeline, _, err := compiler.CompileLite(ctx, p.GetData(), ruleData, false)
	if err != nil {
		retErr := fmt.Errorf("unable to validate pipeline %s: %w", entry, err)

		util.HandleError(c, http.StatusBadRequest, retErr)

		return
	}

	writeOutput(c, pipeline)
}
// capture middleware values
m := c.MustGet("metadata").(*internal.Metadata)
l := c.MustGet("logger").(*logrus.Entry)
p := pipeline.Retrieve(c)
r := repo.Retrieve(c)
u := user.Retrieve(c)
ctx := c.Request.Context()

entry := fmt.Sprintf("%s/%s", r.GetFullName(), p.GetCommit())

Expand All @@ -96,7 +97,7 @@
ruleData := prepareRuleData(c)

// validate the pipeline
pipeline, _, err := compiler.CompileLite(p.GetData(), ruleData, false)
pipeline, _, err := compiler.CompileLite(ctx, p.GetData(), ruleData, false)
if err != nil {
retErr := fmt.Errorf("unable to validate pipeline %s: %w", entry, err)

Expand Down
12 changes: 7 additions & 5 deletions compiler/engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
package compiler

import (
"context"

api "github.com/go-vela/server/api/types"
"github.com/go-vela/server/api/types/settings"
"github.com/go-vela/server/internal"
Expand All @@ -20,12 +22,12 @@ type Engine interface {
// Compile defines a function that produces an executable
// representation of a pipeline from an object. This calls
// Parse internally to convert the object to a yaml configuration.
Compile(interface{}) (*pipeline.Build, *library.Pipeline, error)
Compile(context.Context, interface{}) (*pipeline.Build, *library.Pipeline, error)

// CompileLite defines a function that produces an light executable
// representation of a pipeline from an object. This calls
// Parse internally to convert the object to a yaml configuration.
CompileLite(interface{}, *pipeline.RuleData, bool) (*yaml.Build, *library.Pipeline, error)
CompileLite(context.Context, interface{}, *pipeline.RuleData, bool) (*yaml.Build, *library.Pipeline, error)

// Duplicate defines a function that
// creates a clone of the Engine.
Expand Down Expand Up @@ -71,10 +73,10 @@ type Engine interface {

// ExpandStages defines a function that injects the template
// for each templated step in every stage in a yaml configuration.
ExpandStages(*yaml.Build, map[string]*yaml.Template, *pipeline.RuleData) (*yaml.Build, error)
ExpandStages(context.Context, *yaml.Build, map[string]*yaml.Template, *pipeline.RuleData) (*yaml.Build, error)
// ExpandSteps defines a function that injects the template
// for each templated step in a yaml configuration with the provided template depth.
ExpandSteps(*yaml.Build, map[string]*yaml.Template, *pipeline.RuleData, int) (*yaml.Build, error)
ExpandSteps(context.Context, *yaml.Build, map[string]*yaml.Template, *pipeline.RuleData, int) (*yaml.Build, error)

// Init Compiler Interface Functions

Expand Down Expand Up @@ -147,7 +149,7 @@ type Engine interface {
WithLabels([]string) Engine
// WithPrivateGitHub defines a function that sets
// the private github client in the Engine.
WithPrivateGitHub(string, string) Engine
WithPrivateGitHub(context.Context, string, string) Engine
// GetSettings defines a function that returns new api settings
// with the compiler Engine fields filled.
GetSettings() settings.Compiler
Expand Down
Loading
Loading