From 4a333c777e7353c0811cbaf649acd7adb1e9ad19 Mon Sep 17 00:00:00 2001 From: Lucas Marques Date: Thu, 16 Jan 2025 10:08:48 +0100 Subject: [PATCH] feat: sync repository content with datastore (#467) * feat(repository-controller): implement states and reconciliation loop * feat(datastore): implement client for git bundle storage * fix(repository-controller): run polling every 5 min * feat(repo-controller): failed sync set status to SyncNeeded * feat(gitprovider): implement revision fetch in all providers * feat(repo-polling): implement repo put/fetch from datastore * fix: support unauthenticated git provider * feat(repo-controller): improve logging + use status instead of annotations * fix(repo-controller): fix standard git providers + use git bundle * feat(repo-controller): unify GetGitBundle implementations * fix(repo-controller): add ssh known hosts to controller * feat(repo-controller): wip: reconcile on layer creation * feat(datastore): add HEAD capabilities on git bundles * feat(repo-controller): update repository sync on layer creation * feat(repo-controller): sync now requests by webhook * feat(repo-controller): make repository controller disabled by default --------- Co-authored-by: Luca Corrieri Co-authored-by: Alan --- api/v1alpha1/terraformrepository_types.go | 23 ++ api/v1alpha1/zz_generated.deepcopy.go | 20 ++ cmd/controllers/start.go | 4 +- .../charts/burrito/templates/controllers.yaml | 7 + deploy/charts/burrito/values.yaml | 13 +- internal/annotations/annotations.go | 6 + internal/burrito/config/config.go | 2 + internal/controllers/manager.go | 11 +- .../terraformpullrequest/controller.go | 12 +- .../terraformrepository/conditions.go | 161 ++++++++++++ .../terraformrepository/controller.go | 84 ++++++- .../terraformrepository/polling.go | 56 +++++ .../controllers/terraformrepository/states.go | 232 ++++++++++++++++++ internal/datastore/api/api_test.go | 29 +++ internal/datastore/api/revisions.go | 87 +++++++ internal/datastore/client/client.go | 69 ++++++ internal/datastore/client/mock.go | 27 +- internal/datastore/datastore.go | 3 + internal/datastore/storage/azure/azure.go | 17 ++ internal/datastore/storage/common.go | 25 +- internal/datastore/storage/gcs/gcs.go | 11 + internal/datastore/storage/mock/mock.go | 11 + internal/datastore/storage/s3/s3.go | 14 ++ internal/utils/gitprovider/common/bundle.go | 89 +++++++ internal/utils/gitprovider/github/github.go | 88 ++++--- .../utils/gitprovider/github/github_test.go | 4 +- internal/utils/gitprovider/gitlab/gitlab.go | 65 +++-- .../utils/gitprovider/gitlab/gitlab_test.go | 4 +- internal/utils/gitprovider/mock/mock.go | 11 + .../utils/gitprovider/standard/standard.go | 81 +++++- internal/utils/gitprovider/types/types.go | 3 + internal/utils/typeutils/typeutils.go | 8 + internal/webhook/event/common.go | 2 +- internal/webhook/event/event_test.go | 88 +++---- internal/webhook/event/pullrequest.go | 23 +- internal/webhook/event/push.go | 15 +- ...orm.padok.cloud_terraformrepositories.yaml | 19 ++ manifests/install.yaml | 19 ++ 38 files changed, 1284 insertions(+), 159 deletions(-) create mode 100644 internal/controllers/terraformrepository/conditions.go create mode 100644 internal/controllers/terraformrepository/polling.go create mode 100644 internal/controllers/terraformrepository/states.go create mode 100644 internal/datastore/api/revisions.go create mode 100644 internal/utils/gitprovider/common/bundle.go create mode 100644 internal/utils/typeutils/typeutils.go diff --git a/api/v1alpha1/terraformrepository_types.go b/api/v1alpha1/terraformrepository_types.go index f36a488f..bffe3cb9 100644 --- a/api/v1alpha1/terraformrepository_types.go +++ b/api/v1alpha1/terraformrepository_types.go @@ -46,12 +46,35 @@ type TerraformRepositoryRepository struct { // TerraformRepositoryStatus defines the observed state of TerraformRepository type TerraformRepositoryStatus struct { + State string `json:"state,omitempty"` + Branches []BranchState `json:"branches,omitempty"` Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type" protobuf:"bytes,1,rep,name=conditions"` } +// BranchState describes the sync state of a branch +type BranchState struct { + Name string `json:"name,omitempty"` + LatestRev string `json:"latestRev,omitempty"` + LastSyncDate string `json:"lastSyncDate,omitempty"` + LastSyncStatus string `json:"lastSyncStatus,omitempty"` +} + +// GetBranchState searches for a branch with the specified name in the given slice of BranchState. +// It returns a pointer to the BranchState if found, along with a boolean indicating success. +// If the branch is not found, it returns nil and false. +func GetBranchState(name string, branches []BranchState) (*BranchState, bool) { + for _, branch := range branches { + if branch.Name == name { + return &branch, true + } + } + return nil, false +} + // +kubebuilder:object:root=true // +kubebuilder:resource:shortName=repositories;repository;repo;tfrs;tfr; // +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="State",type=string,JSONPath=`.status.state` // +kubebuilder:printcolumn:name="URL",type=string,JSONPath=`.spec.repository.url` // TerraformRepository is the Schema for the terraformrepositories API type TerraformRepository struct { diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index f995ae40..4098a6df 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -56,6 +56,21 @@ func (in *Attempt) DeepCopy() *Attempt { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BranchState) DeepCopyInto(out *BranchState) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BranchState. +func (in *BranchState) DeepCopy() *BranchState { + if in == nil { + return nil + } + out := new(BranchState) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in ExtraArgs) DeepCopyInto(out *ExtraArgs) { { @@ -627,6 +642,11 @@ func (in *TerraformRepositorySpec) DeepCopy() *TerraformRepositorySpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TerraformRepositoryStatus) DeepCopyInto(out *TerraformRepositoryStatus) { *out = *in + if in.Branches != nil { + in, out := &in.Branches, &out.Branches + *out = make([]BranchState, len(*in)) + copy(*out, *in) + } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]metav1.Condition, len(*in)) diff --git a/cmd/controllers/start.go b/cmd/controllers/start.go index 7d596949..f4c365fe 100644 --- a/cmd/controllers/start.go +++ b/cmd/controllers/start.go @@ -27,10 +27,12 @@ func buildControllersStartCmd(app *burrito.App) *cobra.Command { defaultOnErrorTimer, _ := time.ParseDuration("10s") defaultWaitActionTimer, _ := time.ParseDuration("5s") defaultFailureGracePeriod, _ := time.ParseDuration("15s") + defaultRepositorySyncTimer, _ := time.ParseDuration("5m") cmd.Flags().StringSliceVar(&app.Config.Controller.Namespaces, "namespaces", []string{"burrito-system"}, "list of namespaces to watch") - cmd.Flags().StringSliceVar(&app.Config.Controller.Types, "types", []string{"layer", "repository", "run", "pullrequest"}, "list of controllers to start") + cmd.Flags().StringArrayVar(&app.Config.Controller.Types, "types", []string{"layer", "run", "pullrequest"}, "list of controllers to start") cmd.Flags().DurationVar(&app.Config.Controller.Timers.DriftDetection, "drift-detection-period", defaultDriftDetectionTimer, "period between two plans. Must end with s, m or h.") + cmd.Flags().DurationVar(&app.Config.Controller.Timers.RepositorySync, "repository-sync-period", defaultRepositorySyncTimer, "period between two repository sync. Must end with s, m or h.") cmd.Flags().DurationVar(&app.Config.Controller.Timers.OnError, "on-error-period", defaultOnErrorTimer, "period between two runners launch when an error occurred in the controllers. Must end with s, m or h.") cmd.Flags().DurationVar(&app.Config.Controller.Timers.WaitAction, "wait-action-period", defaultWaitActionTimer, "period between two runners when a layer is locked. Must end with s, m or h.") cmd.Flags().DurationVar(&app.Config.Controller.Timers.FailureGracePeriod, "failure-grace-period", defaultFailureGracePeriod, "initial time before retry, goes exponential function of number failure. Must end with s, m or h.") diff --git a/deploy/charts/burrito/templates/controllers.yaml b/deploy/charts/burrito/templates/controllers.yaml index c049d350..d6a3d510 100644 --- a/deploy/charts/burrito/templates/controllers.yaml +++ b/deploy/charts/burrito/templates/controllers.yaml @@ -60,6 +60,10 @@ spec: - name: burrito-config mountPath: /etc/burrito readOnly: true + - name: ssh-known-hosts + mountPath: /home/burrito/.ssh/known_hosts + subPath: known_hosts + readOnly: true - name: burrito-token mountPath: /var/run/secrets/token readOnly: true @@ -88,6 +92,9 @@ spec: - name: burrito-config configMap: name: burrito-config + - name: ssh-known-hosts + configMap: + name: burrito-ssh-known-hosts - name: burrito-token projected: sources: diff --git a/deploy/charts/burrito/values.yaml b/deploy/charts/burrito/values.yaml index 2a53aa63..b24ddb80 100644 --- a/deploy/charts/burrito/values.yaml +++ b/deploy/charts/burrito/values.yaml @@ -18,6 +18,8 @@ config: timers: # -- Drift detection interval driftDetection: 10m + # -- Repository polling interval + repositorySync: 5m # -- Duration to wait before retrying on error onError: 10s # -- Duration to wait before retrying on locked layer @@ -28,8 +30,9 @@ config: maxConcurrentReconciles: 1 # -- Maximum number of retries for Terraform operations (plan, apply...) terraformMaxRetries: 3 - # -- Resource types to watch for reconciliation - types: ["layer", "repository", "run", "pullrequest"] + # TODO: enable repository controller by default + # -- Resource types to watch for reconciliation. Note: by default repository controller is disabled as it is not yet fully usable. + types: ["layer", "run", "pullrequest"] leaderElection: # -- Enable/Disable leader election enabled: true @@ -271,7 +274,9 @@ controllers: # -- Environment variables to pass to the Burrito controller container envFrom: [] # -- Environment variables to pass to the Burrito controller container - env: [] + env: + - name: SSH_KNOWN_HOSTS + value: /home/burrito/.ssh/known_hosts # -- Additional volumes extraVolumes: {} # -- Additional volume mounts @@ -314,7 +319,7 @@ server: initialDelaySeconds: 5 periodSeconds: 20 # -- Environment variables to pass to the Burrito server container - env: [] + env: [] # -- Environment variables to pass to the Burrito server container envFrom: [] # -- Additional volumes diff --git a/internal/annotations/annotations.go b/internal/annotations/annotations.go index c4940f5f..7d7e6098 100644 --- a/internal/annotations/annotations.go +++ b/internal/annotations/annotations.go @@ -2,6 +2,7 @@ package annotations import ( "context" + "strings" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -21,6 +22,7 @@ const ( LastBranchCommitDate string = "webhook.terraform.padok.cloud/branch-commit-date" LastRelevantCommit string = "webhook.terraform.padok.cloud/relevant-commit" LastRelevantCommitDate string = "webhook.terraform.padok.cloud/relevant-commit-date" + SyncBranchNow string = "webhook.terraform.padok.cloud/sync-" ForceApply string = "notifications.terraform.padok.cloud/force-apply" AdditionnalTriggerPaths string = "config.terraform.padok.cloud/additionnal-trigger-paths" @@ -28,6 +30,10 @@ const ( SyncNow string = "api.terraform.padok.cloud/sync-now" ) +func ComputeKeyForSyncBranchNow(branch string) string { + return SyncBranchNow + strings.ReplaceAll(branch, "/", "--") +} + func Add(ctx context.Context, c client.Client, obj client.Object, annotations map[string]string) error { newObj := obj.DeepCopyObject().(client.Object) patch := client.MergeFrom(newObj) diff --git a/internal/burrito/config/config.go b/internal/burrito/config/config.go index 227d218b..da925710 100644 --- a/internal/burrito/config/config.go +++ b/internal/burrito/config/config.go @@ -100,6 +100,7 @@ type ControllerTimers struct { OnError time.Duration `mapstructure:"onError"` WaitAction time.Duration `mapstructure:"waitAction"` FailureGracePeriod time.Duration `mapstructure:"failureGracePeriod"` + RepositorySync time.Duration `mapstructure:"repositorySync"` } type RepositoryConfig struct { @@ -233,6 +234,7 @@ func TestConfig() *Config { WaitAction: 5 * time.Minute, FailureGracePeriod: 15 * time.Second, OnError: 1 * time.Minute, + RepositorySync: 5 * time.Minute, }, }, Runner: RunnerConfig{ diff --git a/internal/controllers/manager.go b/internal/controllers/manager.go index a486517f..15233ab1 100644 --- a/internal/controllers/manager.go +++ b/internal/controllers/manager.go @@ -108,6 +108,8 @@ func (c *Controllers) Exec() { panic(err.Error()) } + log.Infof("starting these controllers: %v", c.config.Controller.Types) + for _, ctrlType := range c.config.Controller.Types { switch ctrlType { case "layer": @@ -123,10 +125,11 @@ func (c *Controllers) Exec() { log.Infof("layer controller started successfully") case "repository": if err = (&terraformrepository.Reconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Recorder: mgr.GetEventRecorderFor("Burrito"), - Config: c.config, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("Burrito"), + Config: c.config, + Datastore: datastoreClient, }).SetupWithManager(mgr); err != nil { log.Fatalf("unable to create repository controller: %s", err) } diff --git a/internal/controllers/terraformpullrequest/controller.go b/internal/controllers/terraformpullrequest/controller.go index be1f4943..e1e7e3d5 100644 --- a/internal/controllers/terraformpullrequest/controller.go +++ b/internal/controllers/terraformpullrequest/controller.go @@ -3,7 +3,6 @@ package terraformpullrequest import ( "context" "fmt" - "strconv" "github.com/google/go-cmp/cmp" "github.com/padok-team/burrito/internal/burrito/config" @@ -24,6 +23,7 @@ import ( configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1" "github.com/padok-team/burrito/internal/utils/gitprovider" gt "github.com/padok-team/burrito/internal/utils/gitprovider/types" + "github.com/padok-team/burrito/internal/utils/typeutils" ) // Reconciler reconciles a TerraformPullRequest object @@ -147,7 +147,6 @@ func (r *Reconciler) initializeProvider(ctx context.Context, repository *configv log.Debugf("no secret configured for repository %s/%s, skipping provider initialization", repository.Namespace, repository.Name) return nil, nil } - log.Infof("KUBE API REQUEST: getting secret %s/%s", repository.Namespace, repository.Spec.Repository.SecretName) secret := &corev1.Secret{} err := r.Client.Get(ctx, types.NamespacedName{ Name: repository.Spec.Repository.SecretName, @@ -158,9 +157,9 @@ func (r *Reconciler) initializeProvider(ctx context.Context, repository *configv return nil, err } config := gitprovider.Config{ - AppID: parseSecretInt64(secret.Data["githubAppId"]), + AppID: typeutils.ParseSecretInt64(secret.Data["githubAppId"]), URL: repository.Spec.Repository.Url, - AppInstallationID: parseSecretInt64(secret.Data["githubAppInstallationId"]), + AppInstallationID: typeutils.ParseSecretInt64(secret.Data["githubAppInstallationId"]), AppPrivateKey: string(secret.Data["githubAppPrivateKey"]), GitHubToken: string(secret.Data["githubToken"]), GitLabToken: string(secret.Data["gitlabToken"]), @@ -205,8 +204,3 @@ func (r *Reconciler) initializeDefaultProviders() error { } return nil } - -func parseSecretInt64(data []byte) int64 { - v, _ := strconv.ParseInt(string(data), 10, 64) - return v -} diff --git a/internal/controllers/terraformrepository/conditions.go b/internal/controllers/terraformrepository/conditions.go new file mode 100644 index 00000000..0df32417 --- /dev/null +++ b/internal/controllers/terraformrepository/conditions.go @@ -0,0 +1,161 @@ +package terraformrepository + +import ( + "context" + "fmt" + "time" + + configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1" + "github.com/padok-team/burrito/internal/annotations" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// Add newly found branches in the repository's branch state object +func mergeBranchesWithBranchState(found []string, branchStates []configv1alpha1.BranchState) []configv1alpha1.BranchState { + for _, branch := range found { + if _, ok := configv1alpha1.GetBranchState(branch, branchStates); !ok { + branchStates = append(branchStates, configv1alpha1.BranchState{ + Name: branch, + }) + } + } + return branchStates +} + +func isSyncNowRequested(repo *configv1alpha1.TerraformRepository, branch string, lastSyncDate time.Time) (bool, error) { + if syncNow, ok := repo.Annotations[annotations.ComputeKeyForSyncBranchNow(branch)]; ok { + syncNowDate, err := time.Parse(time.UnixDate, syncNow) + if err != nil { + return false, err + } + if syncNowDate.After(lastSyncDate) { + return true, nil + } + } + + return false, nil +} + +// IsLastSyncTooOld checks if the last sync was too long ago for at least one of the branches tracked by the repository +func (r *Reconciler) IsLastSyncTooOld(repo *configv1alpha1.TerraformRepository) (metav1.Condition, bool) { + condition := metav1.Condition{ + Type: "IsLastSyncTooOld", + ObservedGeneration: repo.GetObjectMeta().GetGeneration(), + Status: metav1.ConditionUnknown, + LastTransitionTime: metav1.NewTime(time.Now()), + } + + layerBranches, err := r.retrieveLayerBranches(context.Background(), repo) + if err != nil { + condition.Reason = "ErrorListingLayers" + condition.Message = err.Error() + condition.Status = metav1.ConditionTrue + return condition, true + } + + if len(layerBranches) == 0 { + condition.Reason = "NoBranches" + condition.Message = "No branches managed by this repository, no layers found" + condition.Status = metav1.ConditionFalse + return condition, false + } + + branchStates := repo.Status.Branches + branchStates = mergeBranchesWithBranchState(layerBranches, branchStates) + + for _, branch := range branchStates { + // If no sync has ever happened for this branch, we need one + if branch.LastSyncDate == "" { + condition.Reason = "NoSyncYet" + condition.Message = fmt.Sprintf("Repository has never been synced for branch %s", branch.Name) + condition.Status = metav1.ConditionTrue + return condition, true + } + + lastSync, err := time.Parse(time.UnixDate, branch.LastSyncDate) + if err != nil { + condition.Reason = "InvalidSyncDate" + condition.Message = fmt.Sprintf("Invalid last sync date format for branch %s: %v", branch.Name, err) + condition.Status = metav1.ConditionTrue + return condition, true + } + syncNow, err := isSyncNowRequested(repo, branch.Name, lastSync) + if err != nil { + condition.Reason = "InvalidSyncNowDate" + condition.Message = fmt.Sprintf("Invalid sync now date in annotation %s: %v", annotations.ComputeKeyForSyncBranchNow(branch.Name), err) + condition.Status = metav1.ConditionTrue + return condition, true + } + if syncNow { + condition.Reason = "SyncNowRequested" + condition.Message = fmt.Sprintf("Branch %s has been requested for sync", branch.Name) + condition.Status = metav1.ConditionTrue + return condition, true + } + + nextSyncTime := lastSync.Add(r.Config.Controller.Timers.RepositorySync) + now := time.Now() + + if nextSyncTime.Before(now) { + condition.Reason = "SyncTooOld" + condition.Message = fmt.Sprintf("Last sync for %s was more than %s ago", branch.Name, r.Config.Controller.Timers.RepositorySync) + condition.Status = metav1.ConditionTrue + return condition, true + } + } + + condition.Reason = "SyncRecent" + condition.Message = fmt.Sprintf("Last sync for all branches was less than %s ago", r.Config.Controller.Timers.RepositorySync) + condition.Status = metav1.ConditionFalse + return condition, false +} + +// HasLastSyncFailed checks if the last sync failed +// A sync can fail if at least one of the refs managed by burrito could not be synced with the datastore +func (r *Reconciler) HasLastSyncFailed(repo *configv1alpha1.TerraformRepository) (metav1.Condition, bool) { + condition := metav1.Condition{ + Type: "HasLastSyncFailed", + ObservedGeneration: repo.GetObjectMeta().GetGeneration(), + Status: metav1.ConditionUnknown, + LastTransitionTime: metav1.NewTime(time.Now()), + } + + layerBranches, err := r.retrieveLayerBranches(context.Background(), repo) + if err != nil { + condition.Reason = "ErrorListingLayers" + condition.Message = err.Error() + condition.Status = metav1.ConditionTrue + return condition, true + } + + if len(layerBranches) == 0 { + condition.Reason = "NoBranches" + condition.Message = "No branches managed by this repository, no layers found" + condition.Status = metav1.ConditionFalse + return condition, false + } + + branchStates := repo.Status.Branches + branchStates = mergeBranchesWithBranchState(layerBranches, branchStates) + + for _, branch := range branchStates { + if branch.LastSyncStatus == "" { + condition.Reason = "NoSyncYet" + condition.Message = fmt.Sprintf("Repository has never been synced on branch %s yet", branch.Name) + condition.Status = metav1.ConditionTrue + return condition, false + } + + if branch.LastSyncStatus == SyncStatusFailed { + condition.Reason = "SyncFailed" + condition.Message = fmt.Sprintf("Last sync failed for branch %s", branch.Name) + condition.Status = metav1.ConditionTrue + return condition, true + } + } + + condition.Reason = "SyncSucceeded" + condition.Message = "Last sync succeeded for all branches" + condition.Status = metav1.ConditionFalse + return condition, false +} diff --git a/internal/controllers/terraformrepository/controller.go b/internal/controllers/terraformrepository/controller.go index 81663c1e..bcdc0578 100644 --- a/internal/controllers/terraformrepository/controller.go +++ b/internal/controllers/terraformrepository/controller.go @@ -19,23 +19,35 @@ package terraformrepository import ( "context" + "github.com/google/go-cmp/cmp" log "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1" "github.com/padok-team/burrito/internal/burrito/config" + datastore "github.com/padok-team/burrito/internal/datastore/client" + "github.com/padok-team/burrito/internal/utils/gitprovider" ) // RepositoryReconciler reconciles a TerraformRepository object type Reconciler struct { client.Client - Scheme *runtime.Scheme - Recorder record.EventRecorder - Config *config.Config + Scheme *runtime.Scheme + Recorder record.EventRecorder + Config *config.Config + Providers map[string]gitprovider.Provider + Datastore datastore.Client } //+kubebuilder:rbac:groups=config.terraform.padok.cloud,resources=terraformrepositories,verbs=get;list;watch;create;update;patch;delete @@ -52,17 +64,77 @@ type Reconciler struct { // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.13.0/pkg/reconcile func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - _ = log.WithContext(ctx) + log := log.WithContext(ctx) + log.Infof("starting reconciliation for repository %s/%s ...", req.Namespace, req.Name) - // TODO(user): your logic here + // fetch the TerraformRepository instance + repository := &configv1alpha1.TerraformRepository{} + if err := r.Get(ctx, req.NamespacedName, repository); err != nil { + log.Errorf("failed to get TerraformRepository: %s", err) + // If the repository is not found, it might have been deleted + if errors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, client.IgnoreNotFound(err) + } - return ctrl.Result{}, nil + // Get the current state and conditions + state, conditions := r.GetState(ctx, repository) + stateString := getStateString(state) + + // Update status conditions and state + repository.Status.Conditions = conditions + repository.Status.State = stateString + + // Execute the handler + log.Infof("repository %s/%s is in state %s", repository.Namespace, repository.Name, stateString) + result, branchStates := state.getHandler()(ctx, r, repository) + repository.Status.Branches = branchStates + if err := r.Status().Update(ctx, repository); err != nil { + r.Recorder.Event(repository, corev1.EventTypeWarning, "Reconciliation", "Could not update layer status") + log.Errorf("failed to update repository status: %s", err) + } + log.Infof("finished reconciliation cycle for repository %s/%s", repository.Namespace, repository.Name) + return result, nil } // SetupWithManager sets up the controller with the Manager. func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { + r.Providers = make(map[string]gitprovider.Provider) return ctrl.NewControllerManagedBy(mgr). For(&configv1alpha1.TerraformRepository{}). WithOptions(controller.Options{MaxConcurrentReconciles: r.Config.Controller.MaxConcurrentReconciles}). + Watches(&configv1alpha1.TerraformLayer{}, handler.EnqueueRequestsFromMapFunc( + func(ctx context.Context, obj client.Object) []reconcile.Request { + log.Infof("repository controller has detected the following layer creation: %s/%s", obj.GetNamespace(), obj.GetName()) + layer := obj.(*configv1alpha1.TerraformLayer) + return []reconcile.Request{ + {NamespacedName: types.NamespacedName{Namespace: layer.Spec.Repository.Namespace, Name: layer.Spec.Repository.Name}}, + } + }, + )). + WithEventFilter(ignorePredicate()). Complete(r) } + +func ignorePredicate() predicate.Predicate { + return predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + // Ignore updates on TerraformLayer objects, we only watch their creation + if _, ok := e.ObjectNew.(*configv1alpha1.TerraformLayer); ok { + return false + } + // Update only if generation or annotations change, filter out anything else. + // We only need to check generation or annotations change here, because it is only + // updated on spec changes. On the other hand RevisionVersion + // changes also on status changes. We want to omit reconciliation + // for status updates. + return (e.ObjectOld.GetGeneration() != e.ObjectNew.GetGeneration()) || + cmp.Diff(e.ObjectOld.GetAnnotations(), e.ObjectNew.GetAnnotations()) != "" + }, + DeleteFunc: func(e event.DeleteEvent) bool { + // Evaluates to false if the object has been confirmed deleted. + return !e.DeleteStateUnknown + }, + } +} diff --git a/internal/controllers/terraformrepository/polling.go b/internal/controllers/terraformrepository/polling.go new file mode 100644 index 00000000..bd7e2463 --- /dev/null +++ b/internal/controllers/terraformrepository/polling.go @@ -0,0 +1,56 @@ +package terraformrepository + +import ( + "context" + "fmt" + + configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1" + gitCommon "github.com/padok-team/burrito/internal/utils/gitprovider/common" +) + +// getRemoteRevision gets the latest revision (commit sha) for a given ref from the remote repository +func (r *Reconciler) getRemoteRevision(repository *configv1alpha1.TerraformRepository, ref string) (string, error) { + // Get the appropriate provider for the repository + provider, exists := r.Providers[fmt.Sprintf("%s/%s", repository.Namespace, repository.Name)] + if !exists { + return "", fmt.Errorf("provider not found for repository %s/%s", repository.Namespace, repository.Name) + } + rev, err := provider.GetLatestRevisionForRef(repository, ref) + if err != nil { + return "", fmt.Errorf("failed to get latest revision for ref %s: %v", ref, err) + } + return rev, nil +} + +// getRevisionBundle gets the git bundle for a given revision from the remote repository +func (r *Reconciler) getRevisionBundle(repository *configv1alpha1.TerraformRepository, ref string, revision string) ([]byte, error) { + provider, exists := r.Providers[fmt.Sprintf("%s/%s", repository.Namespace, repository.Name)] + if !exists { + return nil, fmt.Errorf("provider not found for repository %s/%s", repository.Namespace, repository.Name) + } + auth, err := provider.GetGitAuth() + if err != nil { + return nil, fmt.Errorf("failed to get git auth for repository %s/%s: %v", repository.Namespace, repository.Name, err) + } + bundle, err := gitCommon.GetGitBundle(repository, ref, revision, auth) + if err != nil { + return nil, fmt.Errorf("failed to get revision bundle for ref %s: %v", ref, err) + } + return bundle, nil +} + +// retrieveLayerBranches returns the list of refs (branches and tags) that are managed by burrito for a specific repository +func (r *Reconciler) retrieveLayerBranches(ctx context.Context, repository *configv1alpha1.TerraformRepository) ([]string, error) { + // get all layers that depends on the repository (layer.spec.repository.name == repository.name) + layers := &configv1alpha1.TerraformLayerList{} + if err := r.List(ctx, layers); err != nil { + return nil, err + } + refs := []string{} + for _, layer := range layers.Items { + if layer.Spec.Repository.Name == repository.Name { + refs = append(refs, layer.Spec.Branch) + } + } + return refs, nil +} diff --git a/internal/controllers/terraformrepository/states.go b/internal/controllers/terraformrepository/states.go new file mode 100644 index 00000000..05ad7205 --- /dev/null +++ b/internal/controllers/terraformrepository/states.go @@ -0,0 +1,232 @@ +package terraformrepository + +import ( + "context" + "fmt" + "strings" + "time" + + configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1" + "github.com/padok-team/burrito/internal/utils/gitprovider" + gt "github.com/padok-team/burrito/internal/utils/gitprovider/types" + "github.com/padok-team/burrito/internal/utils/typeutils" + log "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" +) + +const ( + SyncStatusSuccess string = "success" + SyncStatusFailed string = "failed" +) + +type Handler func(context.Context, *Reconciler, *configv1alpha1.TerraformRepository) (ctrl.Result, []configv1alpha1.BranchState) + +type State interface { + getHandler() Handler +} + +func (r *Reconciler) GetState(ctx context.Context, repository *configv1alpha1.TerraformRepository) (State, []metav1.Condition) { + log := log.WithContext(ctx) + c1, IsLastSyncTooOld := r.IsLastSyncTooOld(repository) + c2, HasLastSyncFailed := r.HasLastSyncFailed(repository) + conditions := []metav1.Condition{c1, c2} + + if IsLastSyncTooOld || HasLastSyncFailed { + log.Infof("repository %s needs to be synced", repository.Name) + return &SyncNeeded{}, conditions + } + + log.Infof("repository %s is in sync with remote", repository.Name) + return &Synced{}, conditions +} + +type SyncNeeded struct{} + +func (s *SyncNeeded) getHandler() Handler { + return func(ctx context.Context, r *Reconciler, repository *configv1alpha1.TerraformRepository) (ctrl.Result, []configv1alpha1.BranchState) { + log := log.WithContext(ctx) + branchStates := repository.Status.Branches + // Initialize git providers for the repository if needed + if _, ok := r.Providers[fmt.Sprintf("%s/%s", repository.Namespace, repository.Name)]; !ok { + provider, err := r.initializeProvider(ctx, repository) + if err != nil { + log.Errorf("could not initialize provider for repository %s: %s", repository.Name, err) + return ctrl.Result{}, branchStates + } + if provider != nil { + log.Infof("initialized git provider for repository %s/%s", repository.Namespace, repository.Name) + r.Providers[fmt.Sprintf("%s/%s", repository.Namespace, repository.Name)] = provider + } + } + + // Update the list of layer branches by querying the TerraformLayer resources + layerBranches, err := r.retrieveLayerBranches(ctx, repository) + if err != nil { + r.Recorder.Event(repository, corev1.EventTypeWarning, "Reconciliation", "Failed to list managed branches") + log.Errorf("failed to list managed branches: %s", err) + return ctrl.Result{}, branchStates + } + if len(layerBranches) == 0 { + log.Warningf("no managed branches found for repository %s/%s, have you created TerraformLayer resources?", repository.Namespace, repository.Name) + return ctrl.Result{RequeueAfter: r.Config.Controller.Timers.WaitAction}, []configv1alpha1.BranchState{} + } + + // add in branchStates branches that were not previously managed + branchStates = mergeBranchesWithBranchState(layerBranches, branchStates) + + // Update datastore with latest revisions for each ref that needs to be synced + var syncError error + for _, branch := range branchStates { + // Filter out branches that have been synced succesfully recently or do not have been requested to sync now + if lastSync, err := time.Parse(time.UnixDate, branch.LastSyncDate); err == nil { + syncNow, err := isSyncNowRequested(repository, branch.Name, lastSync) + if err != nil { + r.Recorder.Event(repository, corev1.EventTypeWarning, "Reconciliation", fmt.Sprintf("Failed to parse sync now annotation for ref %s", branch.Name)) + continue + } + nextSyncTime := lastSync.Add(r.Config.Controller.Timers.RepositorySync) + now := time.Now() + if !syncNow && !nextSyncTime.Before(now) && branch.LastSyncStatus == SyncStatusSuccess { + continue + } + } + + latestRev, err := r.getRemoteRevision(repository, branch.Name) + if err != nil { + r.Recorder.Event(repository, corev1.EventTypeWarning, "Reconciliation", fmt.Sprintf("Failed to get remote revision for ref %s", branch.Name)) + log.Errorf("failed to get remote revision for ref %s: %s", branch.Name, err) + syncError = err + branchStates = updateBranchState(branchStates, branch.Name, "", SyncStatusFailed) + continue + } + log.Infof("latest revision for repository %s/%s ref:%s is %s", repository.Namespace, repository.Name, branch, latestRev) + + isSynced, err := r.Datastore.CheckGitBundle(repository.Namespace, repository.Name, branch.Name, latestRev) + if err != nil { + r.Recorder.Event(repository, corev1.EventTypeWarning, "Reconciliation", fmt.Sprintf("Failed to check stored revision for ref %s", branch.Name)) + log.Errorf("failed to check stored revision for ref %s: %s", branch.Name, err) + syncError = err + branchStates = updateBranchState(branchStates, branch.Name, latestRev, SyncStatusFailed) + continue + } + + if isSynced { + log.Infof("repository %s/%s is in sync with remote for ref %s: rev %s", repository.Namespace, repository.Name, branch.Name, latestRev) + branchStates = updateBranchState(branchStates, branch.Name, latestRev, SyncStatusSuccess) + continue + } else { + log.Infof("repository %s/%s is out of sync with remote for ref %s. Syncing...", repository.Namespace, repository.Name, branch.Name) + bundle, err := r.getRevisionBundle(repository, branch.Name, latestRev) + if err != nil { + r.Recorder.Event(repository, corev1.EventTypeWarning, "Reconciliation", fmt.Sprintf("Failed to get revision bundle for ref %s", branch.Name)) + log.Errorf("failed to get revision bundle for ref %s: %s", branch.Name, err) + syncError = err + branchStates = updateBranchState(branchStates, branch.Name, latestRev, SyncStatusFailed) + continue + } + + err = r.Datastore.PutGitBundle(repository.Namespace, repository.Name, branch.Name, latestRev, bundle) + if err != nil { + r.Recorder.Event(repository, corev1.EventTypeWarning, "Reconciliation", fmt.Sprintf("Failed to store revision for ref %s", branch.Name)) + log.Errorf("failed to store revision for ref %s: %s", branch.Name, err) + syncError = err + branchStates = updateBranchState(branchStates, branch.Name, latestRev, SyncStatusFailed) + continue + } + log.Infof("stored new bundle for repository %s/%s ref:%s revision:%s", repository.Namespace, repository.Name, branch.Name, latestRev) + branchStates = updateBranchState(branchStates, branch.Name, latestRev, SyncStatusSuccess) + } + } + if syncError != nil { + return ctrl.Result{}, branchStates + } + + r.Recorder.Event(repository, corev1.EventTypeNormal, "Reconciliation", "Repository sync completed") + return ctrl.Result{RequeueAfter: r.Config.Controller.Timers.WaitAction}, branchStates + } +} + +type Synced struct{} + +func (s *Synced) getHandler() Handler { + return func(ctx context.Context, r *Reconciler, repository *configv1alpha1.TerraformRepository) (ctrl.Result, []configv1alpha1.BranchState) { + r.Recorder.Event(repository, corev1.EventTypeNormal, "Reconciliation", "Repository is in sync with remote") + return ctrl.Result{RequeueAfter: r.Config.Controller.Timers.RepositorySync}, repository.Status.Branches + } +} + +func getStateString(state State) string { + t := strings.Split(fmt.Sprintf("%T", state), ".") + return t[len(t)-1] +} + +func updateBranchState(branchStates []configv1alpha1.BranchState, branch, rev, status string) []configv1alpha1.BranchState { + for i, b := range branchStates { + if b.Name == branch { + branchStates[i].LastSyncDate = time.Now().Format(time.UnixDate) + branchStates[i].LatestRev = rev + branchStates[i].LastSyncStatus = status + return branchStates + } + } + return branchStates +} + +func (r *Reconciler) initializeProvider(ctx context.Context, repository *configv1alpha1.TerraformRepository) (gitprovider.Provider, error) { + if repository.Spec.Repository.Url == "" { + return nil, fmt.Errorf("no repository URL found in TerraformRepository.spec.repository.url for repository %s. Skipping provider initialization", repository.Name) + } + var config gitprovider.Config + + if repository.Spec.Repository.SecretName != "" { + secret := &corev1.Secret{} + err := r.Client.Get(ctx, types.NamespacedName{ + Name: repository.Spec.Repository.SecretName, + Namespace: repository.Namespace, + }, secret) + if err != nil { + log.Errorf("failed to get credentials secret for repository %s: %s", repository.Name, err) + config = gitprovider.Config{ + URL: repository.Spec.Repository.Url, + } + } else { + config = gitprovider.Config{ + URL: repository.Spec.Repository.Url, + EnableMock: secret.Data["enableMock"] != nil && string(secret.Data["enableMock"]) == "true", + // GitHub App Auth + AppID: typeutils.ParseSecretInt64(secret.Data["githubAppId"]), + AppInstallationID: typeutils.ParseSecretInt64(secret.Data["githubAppInstallationId"]), + AppPrivateKey: string(secret.Data["githubAppPrivateKey"]), + // Token Auth + GitHubToken: string(secret.Data["githubToken"]), + GitLabToken: string(secret.Data["gitlabToken"]), + // Basic Auth + Username: string(secret.Data["username"]), + Password: string(secret.Data["password"]), + // SSH Auth + SSHPrivateKey: string(secret.Data["sshPrivateKey"]), + } + } + } else { + log.Infof("no secret configured for repository %s/%s, using empty config", repository.Namespace, repository.Name) + config = gitprovider.Config{ + URL: repository.Spec.Repository.Url, + } + } + + provider, err := gitprovider.New(config, []string{gt.Capabilities.Clone}) + if err != nil { + log.Errorf("failed to create provider for repository %s: %s", repository.Name, err) + return nil, err + } + + err = provider.Init() + if err != nil { + log.Errorf("failed to initialize provider for repository %s: %s", repository.Name, err) + return nil, err + } + return provider, nil +} diff --git a/internal/datastore/api/api_test.go b/internal/datastore/api/api_test.go index f3069803..32b69d78 100644 --- a/internal/datastore/api/api_test.go +++ b/internal/datastore/api/api_test.go @@ -38,6 +38,7 @@ var _ = BeforeSuite(func() { API.Storage.PutPlan("default", "test1", "test1", "0", "bin", []byte("test1")) API.Storage.PutPlan("default", "test1", "test1", "0", "short", []byte("test1")) API.Storage.PutPlan("default", "test1", "test1", "0", "pretty", []byte("test1")) + API.Storage.PutGitBundle("default", "test1", "main", "abc123", []byte("test-bundle")) e = echo.New() }) @@ -184,6 +185,34 @@ var _ = Describe("Datastore API", func() { }) }) }) + Describe("Revisions", func() { + Describe("Store Revision", func() { + It("should return 200 OK when storing a revision", func() { + body := []byte(`test-bundle`) + context := getContext(http.MethodPut, "/revisions", map[string]string{ + "namespace": "default", + "name": "test1", + "ref": "main", + "revision": "def456", + }, body) + err := API.PutGitBundleHandler(context) + Expect(err).NotTo(HaveOccurred()) + Expect(context.Response().Status).To(Equal(http.StatusOK)) + }) + + It("should return 400 Bad Request when missing parameters", func() { + body := []byte(`test-bundle`) + context := getContext(http.MethodPut, "/revisions", map[string]string{ + "namespace": "default", + "name": "test1", + // missing ref and revision + }, body) + err := API.PutGitBundleHandler(context) + Expect(err).NotTo(HaveOccurred()) + Expect(context.Response().Status).To(Equal(http.StatusBadRequest)) + }) + }) + }) }) Describe("Write", func() { Describe("Logs", func() { diff --git a/internal/datastore/api/revisions.go b/internal/datastore/api/revisions.go new file mode 100644 index 00000000..74b0f86a --- /dev/null +++ b/internal/datastore/api/revisions.go @@ -0,0 +1,87 @@ +package api + +import ( + "fmt" + "io" + "net/http" + + "github.com/labstack/echo/v4" + storageerrors "github.com/padok-team/burrito/internal/datastore/storage/error" +) + +func getRevisionArgs(c echo.Context) (string, string, string, error) { + namespace := c.QueryParam("namespace") + name := c.QueryParam("name") + ref := c.QueryParam("ref") + if namespace == "" || name == "" || ref == "" { + return "", "", "", fmt.Errorf("missing query parameters") + } + return namespace, name, ref, nil +} + +func (a *API) PutGitBundleHandler(c echo.Context) error { + namespace, name, ref, err := getRevisionArgs(c) + if err != nil { + return c.String(http.StatusBadRequest, err.Error()) + } + + revision := c.QueryParam("revision") + if revision == "" { + return c.String(http.StatusBadRequest, "missing revision parameter") + } + + content, err := io.ReadAll(c.Request().Body) + if err != nil { + return c.String(http.StatusBadRequest, "could not read request body: "+err.Error()) + } + + err = a.Storage.PutGitBundle(namespace, name, ref, revision, content) + if err != nil { + c.Logger().Errorf("Could not store revision, there's an issue with the storage backend: %s", err) + return c.String(http.StatusInternalServerError, "could not store revision, there's an issue with the storage backend") + } + + return c.NoContent(http.StatusOK) +} + +func (a *API) HeadGitBundleHandler(c echo.Context) error { + namespace, name, ref, err := getRevisionArgs(c) + if err != nil { + return c.String(http.StatusBadRequest, err.Error()) + } + revision := c.QueryParam("revision") + if revision == "" { + return c.String(http.StatusBadRequest, "missing revision parameter") + } + checksum, err := a.Storage.CheckGitBundle(namespace, name, ref, revision) + if err != nil { + if storageerrors.NotFound(err) { + return c.String(http.StatusNotFound, "No bundle found for this revision") + } + c.Logger().Errorf("Could not get bundle for revision, there's an issue with the storage backend: %s", err) + return c.String(http.StatusInternalServerError, "could not get bundle for revision, there's an issue with the storage backend") + } + + return c.String(http.StatusOK, string(checksum)) +} + +func (a *API) GetGitBundleHandler(c echo.Context) error { + namespace, name, ref, err := getRevisionArgs(c) + if err != nil { + return c.String(http.StatusBadRequest, err.Error()) + } + revision := c.QueryParam("revision") + if revision == "" { + return c.String(http.StatusBadRequest, "missing revision parameter") + } + content, err := a.Storage.GetGitBundle(namespace, name, ref, revision) + if err != nil { + if storageerrors.NotFound(err) { + return c.String(http.StatusNotFound, "No bundle found for this revision") + } + c.Logger().Errorf("Could not get bundle for revision, there's an issue with the storage backend: %s", err) + return c.String(http.StatusInternalServerError, "could not get bundle for revision, there's an issue with the storage backend") + } + + return c.Blob(http.StatusOK, "application/octet-stream", content) +} diff --git a/internal/datastore/client/client.go b/internal/datastore/client/client.go index ef2563e5..f17fa8d8 100644 --- a/internal/datastore/client/client.go +++ b/internal/datastore/client/client.go @@ -24,6 +24,8 @@ type Client interface { PutPlan(namespace string, layer string, run string, attempt string, format string, content []byte) error GetLogs(namespace string, layer string, run string, attempt string) ([]string, error) PutLogs(namespace string, layer string, run string, attempt string, content []byte) error + PutGitBundle(namespace, name, ref, revision string, bundle []byte) error + CheckGitBundle(namespace, name, ref, revision string) (bool, error) } type DefaultClient struct { @@ -194,3 +196,70 @@ func (c *DefaultClient) PutLogs(namespace string, layer string, run string, atte } return nil } + +func (c *DefaultClient) PutGitBundle(namespace, name, ref, revision string, bundle []byte) error { + req, err := c.buildRequest( + "/api/repository/revision/bundle", + url.Values{ + "namespace": {namespace}, + "name": {name}, + "ref": {ref}, + "revision": {revision}, + }, + http.MethodPut, + bytes.NewBuffer(bundle), + ) + if err != nil { + return err + } + + req.Header.Set("Content-Type", "application/octet-stream") + resp, err := c.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + message, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("could not store revision, there's an issue reading the response from datastore: %s", err) + } + return fmt.Errorf("could not store revision, there's an issue with the storage backend: %s", string(message)) + } + + return nil +} + +func (c *DefaultClient) CheckGitBundle(namespace, name, ref, revision string) (bool, error) { + req, err := c.buildRequest( + "/api/repository/revision/bundle", + url.Values{ + "namespace": {namespace}, + "name": {name}, + "ref": {ref}, + "revision": {revision}, + }, + http.MethodHead, + nil, + ) + if err != nil { + return false, err + } + + resp, err := c.client.Do(req) + if err != nil { + return false, err + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNotFound { + return false, nil + } + + if resp.StatusCode != http.StatusOK { + return false, fmt.Errorf("could not check bundle, there's an issue with the storage backend") + } + + return true, nil +} diff --git a/internal/datastore/client/mock.go b/internal/datastore/client/mock.go index 87506820..dcca872a 100644 --- a/internal/datastore/client/mock.go +++ b/internal/datastore/client/mock.go @@ -1,10 +1,21 @@ package client +import ( + "fmt" +) + type MockClient struct { + // Store latest revisions in memory for testing + revisions map[string]string + // Store bundles in memory for testing + bundles map[string][]byte } func NewMockClient() *MockClient { - return &MockClient{} + return &MockClient{ + revisions: make(map[string]string), + bundles: make(map[string][]byte), + } } func (c *MockClient) GetPlan(namespace string, layer string, run string, attempt string, format string) ([]byte, error) { @@ -26,3 +37,17 @@ func (c *MockClient) PutLogs(namespace string, layer string, run string, attempt func (c *MockClient) GetAttempts(namespace string, layer string, run string) (int, error) { return 0, nil } + +func (c *MockClient) PutGitBundle(namespace, name, ref, revision string, bundle []byte) error { + revKey := fmt.Sprintf("%s/%s/%s", namespace, name, ref) + c.revisions[revKey] = revision + + bundleKey := fmt.Sprintf("%s/%s/%s/%s", namespace, name, ref, revision) + c.bundles[bundleKey] = bundle + + return nil +} + +func (c *MockClient) CheckGitBundle(namespace, name, ref, revision string) (bool, error) { + return false, nil +} diff --git a/internal/datastore/datastore.go b/internal/datastore/datastore.go index a5bce542..c6a72567 100644 --- a/internal/datastore/datastore.go +++ b/internal/datastore/datastore.go @@ -49,6 +49,9 @@ func (s *Datastore) Exec() { api.PUT("/logs", s.API.PutLogsHandler) api.GET("/plans", s.API.GetPlanHandler) api.PUT("/plans", s.API.PutPlanHandler) + api.PUT("/repository/revision/bundle", s.API.PutGitBundleHandler) + api.GET("/repository/revision/bundle", s.API.GetGitBundleHandler) + api.HEAD("/repository/revision/bundle", s.API.HeadGitBundleHandler) if s.Config.Datastore.TLS { e.Logger.Fatal(e.StartTLS(s.Config.Datastore.Addr, DefaultCertPath, DefaultKeyPath)) } else { diff --git a/internal/datastore/storage/azure/azure.go b/internal/datastore/storage/azure/azure.go index a6e4e0cb..7b7c8b9b 100644 --- a/internal/datastore/storage/azure/azure.go +++ b/internal/datastore/storage/azure/azure.go @@ -55,6 +55,23 @@ func (a *Azure) Get(key string) ([]byte, error) { return content, nil } +func (a *Azure) Check(key string) ([]byte, error) { + resp, err := a.Client.ServiceClient().NewContainerClient(a.Config.Container).NewBlobClient(key).GetProperties(context.Background(), nil) + if bloberror.HasCode(err, bloberror.BlobNotFound) { + return make([]byte, 0), &errors.StorageError{ + Err: err, + Nil: true, + } + } + if err != nil { + return make([]byte, 0), &errors.StorageError{ + Err: err, + Nil: false, + } + } + return resp.ContentMD5, nil +} + func (a *Azure) Set(key string, value []byte, ttl int) error { _, err := a.Client.UploadBuffer(context.Background(), a.Config.Container, key, value, nil) if err != nil { diff --git a/internal/datastore/storage/common.go b/internal/datastore/storage/common.go index bdb2c4da..20c9f190 100644 --- a/internal/datastore/storage/common.go +++ b/internal/datastore/storage/common.go @@ -20,7 +20,8 @@ const ( PlanJsonFile string = "plan.json" PrettyPlanFile string = "pretty.plan" ShortDiffFile string = "short.diff" - GitBundleFileExtension string = ".tgz" + GitBundleFileExtension string = ".gitbundle" + RevisionFile string = "latest" LayersPrefix string = "layers" RepositoriesPrefix string = "repositories" ) @@ -47,8 +48,8 @@ func computePlanKey(namespace string, layer string, run string, attempt string, return key } -func computeGitBundleKey(namespace string, repository string, branch string, commit string) string { - return fmt.Sprintf("%s/%s/%s/%s/%s%s", RepositoriesPrefix, namespace, repository, branch, commit, GitBundleFileExtension) +func computeGitBundleKey(namespace string, repository string, branch string, revision string) string { + return fmt.Sprintf("%s/%s/%s/%s/%s%s", RepositoriesPrefix, namespace, repository, branch, revision, GitBundleFileExtension) } type Storage struct { @@ -58,6 +59,7 @@ type Storage struct { type StorageBackend interface { Get(key string) ([]byte, error) + Check(key string) ([]byte, error) Set(key string, value []byte, ttl int) error Delete(key string) error List(prefix string) ([]string, error) @@ -124,10 +126,19 @@ func (s *Storage) GetAttempts(namespace string, layer string, run string) (int, return len(attempts), err } -func (s *Storage) GetGitBundle(namespace string, repository string, branch string, commit string) ([]byte, error) { - return s.Backend.Get(computeGitBundleKey(namespace, repository, branch, commit)) +func (s *Storage) GetGitBundle(namespace string, repository string, ref string, commit string) ([]byte, error) { + return s.Backend.Get(computeGitBundleKey(namespace, repository, ref, commit)) } -func (s *Storage) PutGitBundle(namespace string, repository string, branch string, commit string, bundle []byte) error { - return s.Backend.Set(computeGitBundleKey(namespace, repository, branch, commit), bundle, 0) +func (s *Storage) CheckGitBundle(namespace string, repository string, ref string, commit string) ([]byte, error) { + return s.Backend.Check(computeGitBundleKey(namespace, repository, ref, commit)) +} + +func (s *Storage) PutGitBundle(namespace string, repository string, ref string, commit string, bundle []byte) error { + // Store the git bundle + err := s.Backend.Set(computeGitBundleKey(namespace, repository, ref, commit), bundle, 0) + if err != nil { + return fmt.Errorf("failed to store git bundle: %w", err) + } + return nil } diff --git a/internal/datastore/storage/gcs/gcs.go b/internal/datastore/storage/gcs/gcs.go index be9bc3e2..b27c6ea2 100644 --- a/internal/datastore/storage/gcs/gcs.go +++ b/internal/datastore/storage/gcs/gcs.go @@ -61,6 +61,17 @@ func (a *GCS) Set(key string, data []byte, ttl int) error { return nil } +func (a *GCS) Check(key string) ([]byte, error) { + ctx := context.Background() + bucket := a.Client.Bucket(a.Config.Bucket) + obj := bucket.Object(key) + metadata, err := obj.Attrs(ctx) + if err != nil { + return nil, err + } + return metadata.MD5, nil +} + func (a *GCS) Delete(key string) error { ctx := context.Background() bucket := a.Client.Bucket(a.Config.Bucket) diff --git a/internal/datastore/storage/mock/mock.go b/internal/datastore/storage/mock/mock.go index 49072ed9..67fe4c5a 100644 --- a/internal/datastore/storage/mock/mock.go +++ b/internal/datastore/storage/mock/mock.go @@ -33,6 +33,17 @@ func (s *Mock) Set(key string, value []byte, ttl int) error { return nil } +func (s *Mock) Check(key string) ([]byte, error) { + val, ok := s.data[key] + if !ok { + return nil, &errors.StorageError{ + Err: fmt.Errorf("%s", "Not found"), + Nil: true, + } + } + return val, nil +} + func (s *Mock) Delete(key string) error { delete(s.data, key) return nil diff --git a/internal/datastore/storage/s3/s3.go b/internal/datastore/storage/s3/s3.go index 2942cc0f..18cd39a3 100644 --- a/internal/datastore/storage/s3/s3.go +++ b/internal/datastore/storage/s3/s3.go @@ -54,6 +54,20 @@ func (a *S3) Get(key string) ([]byte, error) { return data, nil } +func (a *S3) Check(key string) ([]byte, error) { + input := &storage.HeadObjectInput{ + Bucket: &a.Config.Bucket, + Key: &key, + } + + result, err := a.Client.HeadObject(context.TODO(), input) + if err != nil { + return nil, err + } + + return []byte(*result.ChecksumSHA256), nil +} + func (a *S3) Set(key string, data []byte, ttl int) error { input := &storage.PutObjectInput{ Bucket: &a.Config.Bucket, diff --git a/internal/utils/gitprovider/common/bundle.go b/internal/utils/gitprovider/common/bundle.go new file mode 100644 index 00000000..c88be602 --- /dev/null +++ b/internal/utils/gitprovider/common/bundle.go @@ -0,0 +1,89 @@ +package common + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/transport" + configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1" + log "github.com/sirupsen/logrus" +) + +const ( + WorkingDir = "/tmp/burrito/repositories" + BundleDir = "/tmp/burrito/gitbundles" +) + +func GetGitBundle(repository *configv1alpha1.TerraformRepository, ref string, revision string, auth transport.AuthMethod) ([]byte, error) { + repoKey := fmt.Sprintf("%s-%s-%s", repository.Namespace, repository.Name, strings.ReplaceAll(ref, "/", "--")) + repoDir := filepath.Join(WorkingDir, repoKey) + + // Try to open existing repository + repo, err := git.PlainOpen(repoDir) + if err != nil { + if err != git.ErrRepositoryNotExists { + return nil, fmt.Errorf("failed to open repository %s: %w", repoKey, err) + } + + // Clone if it doesn't exist + log.Infof("Cloning repository %s to %s", repository.Spec.Repository.Url, repoDir) + cloneOpts := &git.CloneOptions{ + URL: repository.Spec.Repository.Url, + Auth: auth, + ReferenceName: plumbing.NewBranchReferenceName(ref), + } + + repo, err = git.PlainClone(repoDir, false, cloneOpts) + if err != nil { + return nil, fmt.Errorf("failed to clone repository %s: %w", repoKey, err) + } + } + + // Fetch latest changes + fetchOpts := &git.FetchOptions{ + Auth: auth, + } + + log.Infof("fetching latest changes for repo %s", repoKey) + err = repo.Fetch(fetchOpts) + if err != nil { + if err == git.NoErrAlreadyUpToDate { + log.Infof("repository %s is already up-to-date", repoKey) + } else { + return nil, fmt.Errorf("failed to fetch latest changes: %w", err) + } + } + + // Create BundleDir if it doesn't exist + if _, err := os.Stat(BundleDir); os.IsNotExist(err) { + if err := os.MkdirAll(BundleDir, 0755); err != nil { + return nil, fmt.Errorf("failed to create BundleDir directory: %v", err) + } + } + bundleDest := filepath.Join(BundleDir, fmt.Sprintf("%s.gitbundle", repoKey)) + bundle, err := createGitBundle(repoDir, bundleDest, ref) + if err != nil { + return nil, fmt.Errorf("failed to create bundle: %w", err) + } + + return bundle, nil +} + +// Create a git bundle with `git bundle create` and return the content as a byte array +func createGitBundle(sourceDir, destination, ref string) ([]byte, error) { + cmd := exec.Command("git", "-C", sourceDir, "bundle", "create", destination, ref) + output, err := cmd.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("failed to create git bundle: %v, output: %s", err, string(output)) + } + data, err := os.ReadFile(destination) + if err != nil { + return nil, fmt.Errorf("failed to read git bundle: %v", err) + } + return data, nil +} diff --git a/internal/utils/gitprovider/github/github.go b/internal/utils/gitprovider/github/github.go index eaabe759..1f56da03 100644 --- a/internal/utils/gitprovider/github/github.go +++ b/internal/utils/gitprovider/github/github.go @@ -14,6 +14,7 @@ import ( "github.com/bradleyfalzon/ghinstallation/v2" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/plumbing/transport/http" wh "github.com/go-playground/webhooks/github" "github.com/google/go-github/v68/github" @@ -175,33 +176,15 @@ func (g *Github) Comment(repository *configv1alpha1.TerraformRepository, pr *con } func (g *Github) Clone(repository *configv1alpha1.TerraformRepository, branch string, repositoryPath string) (*git.Repository, error) { + auth, err := g.GetGitAuth() + if err != nil { + return nil, err + } + cloneOptions := &git.CloneOptions{ ReferenceName: plumbing.NewBranchReferenceName(branch), URL: repository.Spec.Repository.Url, - } - - if g.GitHubClientType == "app" { - token, err := g.itr.Token(context.Background()) - if err != nil { - return nil, fmt.Errorf("error getting GitHub App token: %w", err) - } - cloneOptions.Auth = &http.BasicAuth{ - Username: "x-access-token", - Password: token, - } - cloneOptions.URL = repository.Spec.Repository.Url - } else if g.GitHubClientType == "token" { - cloneOptions.Auth = &http.BasicAuth{ - Username: "x-access-token", - Password: g.Config.GitHubToken, - } - } else if g.GitHubClientType == "basic" { - cloneOptions.Auth = &http.BasicAuth{ - Username: g.Config.Username, - Password: g.Config.Password, - } - } else { - log.Info("No authentication method provided, falling back to unauthenticated clone") + Auth: auth, } log.Infof("Cloning github repository %s on %s branch with github %s authentication", repository.Spec.Repository.Url, branch, g.GitHubClientType) @@ -242,8 +225,8 @@ func (g *Github) GetEventFromWebhookPayload(p interface{}) (event.Event, error) changedFiles = append(changedFiles, commit.Removed...) } e = &event.PushEvent{ - URL: utils.NormalizeUrl(payload.Repository.HTMLURL), - Revision: event.ParseRevision(payload.Ref), + URL: utils.NormalizeUrl(payload.Repository.HTMLURL), + Reference: event.ParseReference(payload.Ref), ChangeInfo: event.ChangeInfo{ ShaBefore: payload.Before, ShaAfter: payload.After, @@ -257,12 +240,12 @@ func (g *Github) GetEventFromWebhookPayload(p interface{}) (event.Event, error) return nil, err } e = &event.PullRequestEvent{ - ID: strconv.FormatInt(payload.PullRequest.Number, 10), - URL: utils.NormalizeUrl(payload.Repository.HTMLURL), - Revision: payload.PullRequest.Head.Ref, - Action: getNormalizedAction(payload.Action), - Base: payload.PullRequest.Base.Ref, - Commit: payload.PullRequest.Head.Sha, + ID: strconv.FormatInt(payload.PullRequest.Number, 10), + URL: utils.NormalizeUrl(payload.Repository.HTMLURL), + Reference: payload.PullRequest.Head.Ref, + Action: getNormalizedAction(payload.Action), + Base: payload.PullRequest.Base.Ref, + Commit: payload.PullRequest.Head.Sha, } default: return nil, errors.New("unsupported Event") @@ -270,6 +253,19 @@ func (g *Github) GetEventFromWebhookPayload(p interface{}) (event.Event, error) return e, nil } +func (g *Github) GetLatestRevisionForRef(repository *configv1alpha1.TerraformRepository, ref string) (string, error) { + owner, repoName := parseGithubUrl(repository.Spec.Repository.Url) + b, _, err := g.Client.Repositories.GetBranch(context.TODO(), owner, repoName, ref, 10) + if err == nil { + return b.Commit.GetSHA(), nil + } + t, _, err := g.Client.Git.GetRef(context.TODO(), owner, repoName, fmt.Sprintf("refs/tags/%s", ref)) + if err == nil { + return t.Object.GetSHA(), nil + } + return "", fmt.Errorf("could not find revision for ref %s: %w", ref, err) +} + func getNormalizedAction(action string) string { switch action { case "opened", "reopened": @@ -304,3 +300,31 @@ func inferBaseURL(repoURL string) (string, GitHubSubscription, error) { return "", GitHubClassic, nil } } + +// GetGitAuth returns the appropriate authentication method based on the GitHub client type +func (g *Github) GetGitAuth() (transport.AuthMethod, error) { + switch g.GitHubClientType { + case "app": + token, err := g.itr.Token(context.Background()) + if err != nil { + return nil, fmt.Errorf("error getting GitHub App token: %w", err) + } + return &http.BasicAuth{ + Username: "x-access-token", + Password: token, + }, nil + case "token": + return &http.BasicAuth{ + Username: "x-access-token", + Password: g.Config.GitHubToken, + }, nil + case "basic": + return &http.BasicAuth{ + Username: g.Config.Username, + Password: g.Config.Password, + }, nil + default: + log.Info("No authentication method provided, falling back to unauthenticated clone") + return nil, nil + } +} diff --git a/internal/utils/gitprovider/github/github_test.go b/internal/utils/gitprovider/github/github_test.go index 86e48491..93767ffe 100644 --- a/internal/utils/gitprovider/github/github_test.go +++ b/internal/utils/gitprovider/github/github_test.go @@ -68,7 +68,7 @@ func TestGithub_GetEventFromWebhookPayload_PushEvent(t *testing.T) { pushEvt := evt.(*event.PushEvent) assert.Equal(t, "https://github.com/padok-team/burrito-examples", pushEvt.URL) - assert.Equal(t, "main", pushEvt.Revision) + assert.Equal(t, "main", pushEvt.Reference) assert.Equal(t, "6f51b4ffd5e3adadfc3ee649d5ea2499472ea33b", pushEvt.ShaBefore) assert.Equal(t, "ca9b6c80ac8fb5cd837ae9b374b79ff33f472558", pushEvt.ShaAfter) assert.ElementsMatch(t, []string{"modules/random-pets/main.tf", "terragrunt/random-pets/test/inputs.hcl", "modules/random-pets/variables.tf"}, pushEvt.Changes) @@ -123,7 +123,7 @@ func TestGithub_GetEventFromWebhookPayload_PullRequestEvent(t *testing.T) { pullRequestEvt := evt.(*event.PullRequestEvent) assert.Equal(t, "20", pullRequestEvt.ID) assert.Equal(t, "https://github.com/padok-team/burrito-examples", pullRequestEvt.URL) - assert.Equal(t, "demo", pullRequestEvt.Revision) + assert.Equal(t, "demo", pullRequestEvt.Reference) assert.Equal(t, "main", pullRequestEvt.Base) assert.Equal(t, "faf5e25402a9bd10f7318c8a2cd984af576c687f", pullRequestEvt.Commit) assert.Equal(t, "opened", pullRequestEvt.Action) diff --git a/internal/utils/gitprovider/gitlab/gitlab.go b/internal/utils/gitprovider/gitlab/gitlab.go index 3f1b2bbd..01ae8f71 100644 --- a/internal/utils/gitprovider/gitlab/gitlab.go +++ b/internal/utils/gitprovider/gitlab/gitlab.go @@ -11,6 +11,7 @@ import ( "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/plumbing/transport/http" wh "github.com/go-playground/webhooks/gitlab" configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1" @@ -133,21 +134,18 @@ func (g *Gitlab) Comment(repository *configv1alpha1.TerraformRepository, pr *con } func (g *Gitlab) Clone(repository *configv1alpha1.TerraformRepository, branch string, repositoryPath string) (*git.Repository, error) { + auth, err := g.GetGitAuth() + if err != nil { + return nil, err + } + cloneOptions := &git.CloneOptions{ ReferenceName: plumbing.NewBranchReferenceName(branch), URL: repository.Spec.Repository.Url, + Auth: auth, } - if g.Config.GitLabToken != "" { - cloneOptions.Auth = &http.BasicAuth{ - Password: g.Config.GitLabToken, - } - } else if g.Config.Username != "" && g.Config.Password != "" { - cloneOptions.Auth = &http.BasicAuth{ - Username: g.Config.Username, - Password: g.Config.Password, - } - } else { + if auth == nil { return nil, errors.New("no valid authentication method provided") } @@ -189,8 +187,8 @@ func (g *Gitlab) GetEventFromWebhookPayload(p interface{}) (event.Event, error) changedFiles = append(changedFiles, commit.Removed...) } e = &event.PushEvent{ - URL: utils.NormalizeUrl(payload.Project.WebURL), - Revision: event.ParseRevision(payload.Ref), + URL: utils.NormalizeUrl(payload.Project.WebURL), + Reference: event.ParseReference(payload.Ref), ChangeInfo: event.ChangeInfo{ ShaBefore: payload.Before, ShaAfter: payload.After, @@ -200,12 +198,12 @@ func (g *Gitlab) GetEventFromWebhookPayload(p interface{}) (event.Event, error) case wh.MergeRequestEventPayload: log.Infof("parsing Gitlab merge request event payload") e = &event.PullRequestEvent{ - ID: strconv.Itoa(int(payload.ObjectAttributes.IID)), - URL: utils.NormalizeUrl(payload.Project.WebURL), - Revision: payload.ObjectAttributes.SourceBranch, - Action: getNormalizedAction(payload.ObjectAttributes.Action), - Base: payload.ObjectAttributes.TargetBranch, - Commit: payload.ObjectAttributes.LastCommit.ID, + ID: strconv.Itoa(int(payload.ObjectAttributes.IID)), + URL: utils.NormalizeUrl(payload.Project.WebURL), + Reference: payload.ObjectAttributes.SourceBranch, + Action: getNormalizedAction(payload.ObjectAttributes.Action), + Base: payload.ObjectAttributes.TargetBranch, + Commit: payload.ObjectAttributes.LastCommit.ID, } default: return nil, errors.New("unsupported event") @@ -213,6 +211,20 @@ func (g *Gitlab) GetEventFromWebhookPayload(p interface{}) (event.Event, error) return e, nil } +// Required API scope: api read_api +func (g *Gitlab) GetLatestRevisionForRef(repository *configv1alpha1.TerraformRepository, ref string) (string, error) { + projectID := getGitlabNamespacedName(repository.Spec.Repository.Url) + b, _, err := g.Client.Branches.GetBranch(projectID, ref) + if err == nil { + return b.Commit.ID, nil + } + t, _, err := g.Client.Tags.GetTag(projectID, ref) + if err == nil { + return t.Commit.ID, nil + } + return "", fmt.Errorf("could not find revision for ref %s: %w", ref, err) +} + func getNormalizedAction(action string) string { switch action { case "open", "reopen": @@ -239,3 +251,20 @@ func inferBaseURL(repoURL string) (string, error) { host = strings.TrimPrefix(host, "www.") return fmt.Sprintf("https://%s/api/v4", host), nil } + +// GetGitAuth returns the appropriate authentication method for GitLab +func (g *Gitlab) GetGitAuth() (transport.AuthMethod, error) { + if g.Config.GitLabToken != "" { + return &http.BasicAuth{ + Username: "oauth2", + Password: g.Config.GitLabToken, + }, nil + } else if g.Config.Username != "" && g.Config.Password != "" { + return &http.BasicAuth{ + Username: g.Config.Username, + Password: g.Config.Password, + }, nil + } + log.Info("No authentication method provided, falling back to unauthenticated clone") + return nil, nil +} diff --git a/internal/utils/gitprovider/gitlab/gitlab_test.go b/internal/utils/gitprovider/gitlab/gitlab_test.go index 84757e15..f4334ec0 100644 --- a/internal/utils/gitprovider/gitlab/gitlab_test.go +++ b/internal/utils/gitprovider/gitlab/gitlab_test.go @@ -59,7 +59,7 @@ func TestGitlab_GetEventFromWebhookPayload_PushEvent(t *testing.T) { pushEvt := evt.(*event.PushEvent) assert.Equal(t, "https://gitlab.com/burrito/examples", pushEvt.URL) - assert.Equal(t, "main", pushEvt.Revision) + assert.Equal(t, "main", pushEvt.Reference) assert.Equal(t, "95790bf891e76fee5e1747ab589903a6a1f80f22", pushEvt.ShaBefore) assert.Equal(t, "da1560886d4f094c3e6c9ef40349f7d38b5d27d7", pushEvt.ShaAfter) assert.ElementsMatch(t, []string{"test.hcl", "layer-1/prod.hcl", "layer-2/staging.hcl"}, pushEvt.Changes) @@ -114,7 +114,7 @@ func TestGitlab_GetEventFromWebhookPayload_MergeRequestEvent(t *testing.T) { pullRequestEvt := evt.(*event.PullRequestEvent) assert.Equal(t, "1", pullRequestEvt.ID) assert.Equal(t, "https://example.com/gitlabhq/gitlab-test", pullRequestEvt.URL) - assert.Equal(t, "demo", pullRequestEvt.Revision) + assert.Equal(t, "demo", pullRequestEvt.Reference) assert.Equal(t, "main", pullRequestEvt.Base) assert.Equal(t, "da1560886d4f094c3e6c9ef40349f7d38b5d27d7", pullRequestEvt.Commit) assert.Equal(t, expected, pullRequestEvt.Action) diff --git a/internal/utils/gitprovider/mock/mock.go b/internal/utils/gitprovider/mock/mock.go index d30c2615..3fd1a237 100644 --- a/internal/utils/gitprovider/mock/mock.go +++ b/internal/utils/gitprovider/mock/mock.go @@ -5,6 +5,7 @@ import ( "slices" "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/transport" configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1" "github.com/padok-team/burrito/internal/controllers/terraformpullrequest/comment" "github.com/padok-team/burrito/internal/utils/gitprovider/types" @@ -56,6 +57,11 @@ func (m *Mock) GetChanges(repository *configv1alpha1.TerraformRepository, pr *co return allChangedFiles, nil } +func (m *Mock) GetLatestRevisionForRef(repository *configv1alpha1.TerraformRepository, ref string) (string, error) { + log.Infof("Mock provider latest revision for ref") + return "", nil +} + func (m *Mock) Comment(repository *configv1alpha1.TerraformRepository, pr *configv1alpha1.TerraformPullRequest, comment comment.Comment) error { log.Infof("Mock provider comment posted") return nil @@ -75,3 +81,8 @@ func (m *Mock) GetEventFromWebhookPayload(payload interface{}) (event.Event, err log.Infof("Mock provider webhook event parsed") return nil, nil } + +func (m *Mock) GetGitAuth() (transport.AuthMethod, error) { + log.Infof("Mock provider git authentication") + return nil, nil +} diff --git a/internal/utils/gitprovider/standard/standard.go b/internal/utils/gitprovider/standard/standard.go index d52d2173..c2cc509c 100644 --- a/internal/utils/gitprovider/standard/standard.go +++ b/internal/utils/gitprovider/standard/standard.go @@ -6,9 +6,12 @@ import ( "strings" "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/go-git/go-git/v5/plumbing/transport/ssh" + "github.com/go-git/go-git/v5/storage/memory" configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1" "github.com/padok-team/burrito/internal/controllers/terraformpullrequest/comment" "github.com/padok-team/burrito/internal/utils/gitprovider/types" @@ -35,6 +38,44 @@ func (s *Standard) GetChanges(repository *configv1alpha1.TerraformRepository, pr return nil, fmt.Errorf("GetChanges not supported for standard git provider. Provide a specific credentials for providers such as GitHub or GitLab") } +func (s *Standard) GetLatestRevisionForRef(repository *configv1alpha1.TerraformRepository, ref string) (string, error) { + auth, err := s.GetGitAuth() + if err != nil { + return "", fmt.Errorf("failed to get git auth: %w", err) + } + + // Create an in-memory remote + remote := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{ + Name: "origin", + URLs: []string{repository.Spec.Repository.Url}, + }) + + // List references on the remote (equivalent to `git ls-remote `) + refs, err := remote.List(&git.ListOptions{ + Auth: auth, + }) + if err != nil { + return "", fmt.Errorf("failed to list references: %v", err) + } + + candidates := []string{ + "refs/heads/" + ref, + "refs/tags/" + ref, + ref, // in case someone passes the full ref already + } + + // Look for the ref in the remote’s references + for _, c := range candidates { + for _, r := range refs { + if r.Name().String() == c { + return r.Hash().String(), nil + } + } + } + + return "", fmt.Errorf("unable to find commit SHA for ref %q in %q", ref, repository.Spec.Repository.Url) +} + func (s *Standard) Comment(repository *configv1alpha1.TerraformRepository, pr *configv1alpha1.TerraformPullRequest, comment comment.Comment) error { return fmt.Errorf("Comment not supported for standard git provider. Provide a specific credentials for providers such as GitHub or GitLab") } @@ -44,24 +85,18 @@ func (s *Standard) CreatePullRequest(repository *configv1alpha1.TerraformReposit } func (g *Standard) Clone(repository *configv1alpha1.TerraformRepository, branch string, repositoryPath string) (*git.Repository, error) { + auth, err := g.GetGitAuth() + if err != nil { + return nil, err + } + cloneOptions := &git.CloneOptions{ ReferenceName: plumbing.NewBranchReferenceName(branch), URL: repository.Spec.Repository.Url, + Auth: auth, } - isSSH := strings.HasPrefix(repository.Spec.Repository.Url, "git@") || strings.Contains(repository.Spec.Repository.Url, "ssh://") - if isSSH && g.Config.SSHPrivateKey != "" { - publicKeys, err := ssh.NewPublicKeys("git", []byte(g.Config.SSHPrivateKey), "") - if err != nil { - return nil, err - } - cloneOptions.Auth = publicKeys - } else if g.Config.Username != "" && g.Config.Password != "" { - cloneOptions.Auth = &http.BasicAuth{ - Username: g.Config.Username, - Password: g.Config.Password, - } - } else { + if auth == nil { log.Info("No authentication method provided, falling back to unauthenticated clone") } @@ -81,3 +116,23 @@ func (m *Standard) ParseWebhookPayload(payload *nethttp.Request) (interface{}, b func (m *Standard) GetEventFromWebhookPayload(payload interface{}) (event.Event, error) { return nil, fmt.Errorf("GetEventFromWebhookPayload not supported for standard git provider. Provide a specific credentials for providers such as GitHub or GitLab") } + +func (s *Standard) GetGitAuth() (transport.AuthMethod, error) { + repoURL := s.Config.URL + isSSH := strings.HasPrefix(repoURL, "git@") || strings.Contains(repoURL, "ssh://") + + if isSSH && s.Config.SSHPrivateKey != "" { + publicKeys, err := ssh.NewPublicKeys("git", []byte(s.Config.SSHPrivateKey), "") + if err != nil { + return nil, err + } + return publicKeys, nil + } else if s.Config.Username != "" && s.Config.Password != "" { + return &http.BasicAuth{ + Username: s.Config.Username, + Password: s.Config.Password, + }, nil + } + log.Info("no authentication method provided, falling back to unauthenticated clone") + return nil, nil +} diff --git a/internal/utils/gitprovider/types/types.go b/internal/utils/gitprovider/types/types.go index e978ca1c..c799b756 100644 --- a/internal/utils/gitprovider/types/types.go +++ b/internal/utils/gitprovider/types/types.go @@ -4,6 +4,7 @@ import ( "net/http" "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/transport" configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1" "github.com/padok-team/burrito/internal/controllers/terraformpullrequest/comment" "github.com/padok-team/burrito/internal/webhook/event" @@ -43,7 +44,9 @@ type Provider interface { InitWebhookHandler() error GetChanges(*configv1alpha1.TerraformRepository, *configv1alpha1.TerraformPullRequest) ([]string, error) Comment(*configv1alpha1.TerraformRepository, *configv1alpha1.TerraformPullRequest, comment.Comment) error + GetGitAuth() (transport.AuthMethod, error) Clone(*configv1alpha1.TerraformRepository, string, string) (*git.Repository, error) + GetLatestRevisionForRef(*configv1alpha1.TerraformRepository, string) (string, error) ParseWebhookPayload(r *http.Request) (interface{}, bool) GetEventFromWebhookPayload(interface{}) (event.Event, error) } diff --git a/internal/utils/typeutils/typeutils.go b/internal/utils/typeutils/typeutils.go new file mode 100644 index 00000000..b9f3f9fb --- /dev/null +++ b/internal/utils/typeutils/typeutils.go @@ -0,0 +1,8 @@ +package typeutils + +import "strconv" + +func ParseSecretInt64(data []byte) int64 { + v, _ := strconv.ParseInt(string(data), 10, 64) + return v +} diff --git a/internal/webhook/event/common.go b/internal/webhook/event/common.go index 9696ee68..78aec5b9 100644 --- a/internal/webhook/event/common.go +++ b/internal/webhook/event/common.go @@ -19,7 +19,7 @@ type Event interface { Handle(client.Client) error } -func ParseRevision(ref string) string { +func ParseReference(ref string) string { refParts := strings.SplitN(ref, "/", 3) return refParts[len(refParts)-1] } diff --git a/internal/webhook/event/event_test.go b/internal/webhook/event/event_test.go index 9e006e91..a602c90b 100644 --- a/internal/webhook/event/event_test.go +++ b/internal/webhook/event/event_test.go @@ -58,8 +58,8 @@ var _ = BeforeSuite(func() { }) var PushEventNoChanges = event.PushEvent{ - URL: "https://github.com/padok-team/burrito-examples", - Revision: "main", + URL: "https://github.com/padok-team/burrito-examples", + Reference: "main", ChangeInfo: event.ChangeInfo{ ShaBefore: "b3231e8771591b3864b3c582e85955c1f76aaded", ShaAfter: "6c193d9cad1ddafdb31ff9f733630da9705bfd64", @@ -70,8 +70,8 @@ var PushEventNoChanges = event.PushEvent{ } var PushEventLayerPathChanges = event.PushEvent{ - URL: "https://github.com/padok-team/burrito-examples", - Revision: "main", + URL: "https://github.com/padok-team/burrito-examples", + Reference: "main", ChangeInfo: event.ChangeInfo{ ShaBefore: "b3231e8771591b3864b3c582e85955c1f76aaded", ShaAfter: "6c193d9cad1ddafdb31ff9f733630da9705bfd64", @@ -82,8 +82,8 @@ var PushEventLayerPathChanges = event.PushEvent{ } var PushEventAdditionalPathChanges = event.PushEvent{ - URL: "https://github.com/padok-team/burrito-examples", - Revision: "main", + URL: "https://github.com/padok-team/burrito-examples", + Reference: "main", ChangeInfo: event.ChangeInfo{ ShaBefore: "b3231e8771591b3864b3c582e85955c1f76aaded", ShaAfter: "6c193d9cad1ddafdb31ff9f733630da9705bfd64", @@ -95,8 +95,8 @@ var PushEventAdditionalPathChanges = event.PushEvent{ } var PushEventMultiplePathChanges = event.PushEvent{ - URL: "https://github.com/padok-team/burrito-examples", - Revision: "main", + URL: "https://github.com/padok-team/burrito-examples", + Reference: "main", ChangeInfo: event.ChangeInfo{ ShaBefore: "b3231e8771591b3864b3c582e85955c1f76aaded", ShaAfter: "6c193d9cad1ddafdb31ff9f733630da9705bfd64", @@ -108,57 +108,57 @@ var PushEventMultiplePathChanges = event.PushEvent{ } var PullRequestOpenedEventNotAffected = event.PullRequestEvent{ - URL: "https://github.com/example/repo", - Revision: "feature/branch", - Base: "main", - Action: "opened", - ID: "42", - Commit: "5b2c5e5c6699bf2bf93138205565b85193996572", + URL: "https://github.com/example/repo", + Reference: "feature/branch", + Base: "main", + Action: "opened", + ID: "42", + Commit: "5b2c5e5c6699bf2bf93138205565b85193996572", } var PullRequestClosedEventNotAffected = event.PullRequestEvent{ - URL: "https://github.com/example/repo", - Revision: "feature/branch", - Base: "main", - Action: "closed", - ID: "42", - Commit: "5b2c5e5c6699bf2bf93138205565b85193996572", + URL: "https://github.com/example/repo", + Reference: "feature/branch", + Base: "main", + Action: "closed", + ID: "42", + Commit: "5b2c5e5c6699bf2bf93138205565b85193996572", } var PullRequestOpenedEventSingleAffected = event.PullRequestEvent{ - URL: "https://github.com/padok-team/burrito-examples", - Revision: "feature/branch", - Base: "main", - Action: "opened", - ID: "42", - Commit: "5b2c5e5c6699bf2bf93138205565b85193996572", + URL: "https://github.com/padok-team/burrito-examples", + Reference: "feature/branch", + Base: "main", + Action: "opened", + ID: "42", + Commit: "5b2c5e5c6699bf2bf93138205565b85193996572", } var PullRequestClosedEventSingleAffected = event.PullRequestEvent{ - URL: "https://github.com/padok-team/burrito-closed-single-pr", - Revision: "feature/branch", - Base: "main", - Action: "closed", - ID: "42", - Commit: "5b2c5e5c6699bf2bf93138205565b85193996572", + URL: "https://github.com/padok-team/burrito-closed-single-pr", + Reference: "feature/branch", + Base: "main", + Action: "closed", + ID: "42", + Commit: "5b2c5e5c6699bf2bf93138205565b85193996572", } var PullRequestOpenedEventMultipleAffected = event.PullRequestEvent{ - URL: "https://github.com/example/other-repo", - Revision: "feature/branch", - Base: "main", - Action: "opened", - ID: "42", - Commit: "5b2c5e5c6699bf2bf93138205565b85193996572", + URL: "https://github.com/example/other-repo", + Reference: "feature/branch", + Base: "main", + Action: "opened", + ID: "42", + Commit: "5b2c5e5c6699bf2bf93138205565b85193996572", } var PullRequestClosedEventMultipleAffected = event.PullRequestEvent{ - URL: "https://github.com/padok-team/burrito-closed-multi-pr", - Revision: "feature/branch", - Base: "main", - Action: "closed", - ID: "42", - Commit: "5b2c5e5c6699bf2bf93138205565b85193996572", + URL: "https://github.com/padok-team/burrito-closed-multi-pr", + Reference: "feature/branch", + Base: "main", + Action: "closed", + ID: "42", + Commit: "5b2c5e5c6699bf2bf93138205565b85193996572", } var _ = Describe("Webhook", func() { diff --git a/internal/webhook/event/pullrequest.go b/internal/webhook/event/pullrequest.go index f3db1d25..6d89b522 100644 --- a/internal/webhook/event/pullrequest.go +++ b/internal/webhook/event/pullrequest.go @@ -16,12 +16,12 @@ import ( ) type PullRequestEvent struct { - URL string - Revision string - Base string - Action string - ID string - Commit string + URL string + Reference string + Base string + Action string + ID string + Commit string } func (e *PullRequestEvent) Handle(c client.Client) error { @@ -36,11 +36,20 @@ func (e *PullRequestEvent) Handle(c client.Client) error { log.Infof("no affected repositories found for pull request event") return nil } + prs := e.generateTerraformPullRequests(affectedRepositories) switch e.Action { case PullRequestOpened: return batchCreatePullRequests(context.TODO(), c, prs) case PullRequestClosed: + // remove annotation from affected repositories + for _, repo := range affectedRepositories { + key := annotations.ComputeKeyForSyncBranchNow(e.Reference) + err := annotations.Remove(context.TODO(), c, &repo, key) + if err != nil { + log.Errorf("could not remove annotation to TerraformRepository %s", err) + } + } return batchDeletePullRequests(context.TODO(), c, prs) default: log.Infof("action %s not supported", e.Action) @@ -84,7 +93,7 @@ func (e *PullRequestEvent) generateTerraformPullRequests(repositories []configv1 }, }, Spec: configv1alpha1.TerraformPullRequestSpec{ - Branch: e.Revision, + Branch: e.Reference, ID: e.ID, Base: e.Base, Repository: configv1alpha1.TerraformLayerRepository{ diff --git a/internal/webhook/event/push.go b/internal/webhook/event/push.go index bf151a27..0b41389e 100644 --- a/internal/webhook/event/push.go +++ b/internal/webhook/event/push.go @@ -14,8 +14,8 @@ import ( ) type PushEvent struct { - URL string - Revision string + URL string + Reference string ChangeInfo Changes []string } @@ -43,8 +43,7 @@ func (e *PushEvent) Handle(c client.Client) error { affectedRepositories := e.getAffectedRepositories(repositories.Items) for _, repo := range affectedRepositories { ann := map[string]string{} - ann[annotations.LastBranchCommit] = e.ChangeInfo.ShaAfter - ann[annotations.LastBranchCommitDate] = date + ann[annotations.ComputeKeyForSyncBranchNow(e.Reference)] = time.Now().Format(time.UnixDate) err := annotations.Add(context.TODO(), c, &repo, ann) if err != nil { log.Errorf("could not add annotation to TerraformRepository %s", err) @@ -54,9 +53,9 @@ func (e *PushEvent) Handle(c client.Client) error { for _, layer := range e.getAffectedLayers(layers.Items, affectedRepositories) { ann := map[string]string{} - log.Printf("evaluating TerraformLayer %s for revision %s", layer.Name, e.Revision) - if layer.Spec.Branch != e.Revision { - log.Infof("branch %s for TerraformLayer %s not matching revision %s", layer.Spec.Branch, layer.Name, e.Revision) + log.Printf("evaluating TerraformLayer %s for revision %s", layer.Name, e.Reference) + if layer.Spec.Branch != e.Reference { + log.Infof("branch %s for TerraformLayer %s not matching revision %s", layer.Spec.Branch, layer.Name, e.Reference) continue } ann[annotations.LastBranchCommit] = e.ChangeInfo.ShaAfter @@ -113,7 +112,7 @@ func (e *PushEvent) getAffectedLayers(allLayers []configv1alpha1.TerraformLayer, func (e *PushEvent) getAffectedPullRequests(prs []configv1alpha1.TerraformPullRequest, affectedRepositories []configv1alpha1.TerraformRepository) []configv1alpha1.TerraformPullRequest { affectedPRs := []configv1alpha1.TerraformPullRequest{} for _, pr := range prs { - if isPRLinkedToAnyRepositories(pr, affectedRepositories) && pr.Spec.Branch == e.Revision { + if isPRLinkedToAnyRepositories(pr, affectedRepositories) && pr.Spec.Branch == e.Reference { affectedPRs = append(affectedPRs, pr) } } diff --git a/manifests/crds/config.terraform.padok.cloud_terraformrepositories.yaml b/manifests/crds/config.terraform.padok.cloud_terraformrepositories.yaml index b5e6a36d..e657c5ba 100644 --- a/manifests/crds/config.terraform.padok.cloud_terraformrepositories.yaml +++ b/manifests/crds/config.terraform.padok.cloud_terraformrepositories.yaml @@ -21,6 +21,9 @@ spec: scope: Namespaced versions: - additionalPrinterColumns: + - jsonPath: .status.state + name: State + type: string - jsonPath: .spec.repository.url name: URL type: string @@ -2259,6 +2262,20 @@ spec: status: description: TerraformRepositoryStatus defines the observed state of TerraformRepository properties: + branches: + items: + description: BranchState describes the sync state of a branch + properties: + lastSyncDate: + type: string + lastSyncStatus: + type: string + latestRev: + type: string + name: + type: string + type: object + type: array conditions: items: description: Condition contains details for one aspect of the current @@ -2315,6 +2332,8 @@ spec: - type type: object type: array + state: + type: string type: object type: object served: true diff --git a/manifests/install.yaml b/manifests/install.yaml index b1496172..09e8bb0e 100644 --- a/manifests/install.yaml +++ b/manifests/install.yaml @@ -2534,6 +2534,9 @@ spec: scope: Namespaced versions: - additionalPrinterColumns: + - jsonPath: .status.state + name: State + type: string - jsonPath: .spec.repository.url name: URL type: string @@ -4772,6 +4775,20 @@ spec: status: description: TerraformRepositoryStatus defines the observed state of TerraformRepository properties: + branches: + items: + description: BranchState describes the sync state of a branch + properties: + lastSyncDate: + type: string + lastSyncStatus: + type: string + latestRev: + type: string + name: + type: string + type: object + type: array conditions: items: description: Condition contains details for one aspect of the current @@ -4828,6 +4845,8 @@ spec: - type type: object type: array + state: + type: string type: object type: object served: true