feat: sync repository content with datastore (#467)

* feat(repository-controller): implement states and reconciliation loop

* feat(datastore): implement client for git bundle storage

* fix(repository-controller): run polling every 5 min

* feat(repo-controller): failed sync set status to SyncNeeded

* feat(gitprovider): implement revision fetch in all providers

* feat(repo-polling): implement repo put/fetch from datastore

* fix: support unauthenticated git provider

* feat(repo-controller): improve logging + use status instead of annotations

* fix(repo-controller): fix standard git providers + use git bundle

* feat(repo-controller): unify GetGitBundle implementations

* fix(repo-controller): add ssh known hosts to controller

* feat(repo-controller): wip: reconcile on layer creation

* feat(datastore): add HEAD capabilities on git bundles

* feat(repo-controller): update repository sync on layer creation

* feat(repo-controller): sync now requests by webhook

* feat(repo-controller): make repository controller disabled by default

---------

Co-authored-by: Luca Corrieri <[email protected]>
Co-authored-by: Alan <[email protected]>

api/v1alpha1/terraformrepository_types.go

@@ -46,12 +46,35 @@ type TerraformRepositoryRepository struct {
 
 // TerraformRepositoryStatus defines the observed state of TerraformRepository
 type TerraformRepositoryStatus struct {
+	State      string             `json:"state,omitempty"`
+	Branches   []BranchState      `json:"branches,omitempty"`
 	Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type" protobuf:"bytes,1,rep,name=conditions"`
 }
 
+// BranchState describes the sync state of a branch
+type BranchState struct {
+	Name           string `json:"name,omitempty"`
+	LatestRev      string `json:"latestRev,omitempty"`
+	LastSyncDate   string `json:"lastSyncDate,omitempty"`
+	LastSyncStatus string `json:"lastSyncStatus,omitempty"`
+}
+
+// GetBranchState searches for a branch with the specified name in the given slice of BranchState.
+// It returns a pointer to the BranchState if found, along with a boolean indicating success.
+// If the branch is not found, it returns nil and false.
+func GetBranchState(name string, branches []BranchState) (*BranchState, bool) {
+	for _, branch := range branches {
+		if branch.Name == name {
+			return &branch, true
+		}
+	}
+	return nil, false
+}
+
 // +kubebuilder:object:root=true
 // +kubebuilder:resource:shortName=repositories;repository;repo;tfrs;tfr;
 // +kubebuilder:subresource:status
+// +kubebuilder:printcolumn:name="State",type=string,JSONPath=`.status.state`
 // +kubebuilder:printcolumn:name="URL",type=string,JSONPath=`.spec.repository.url`
 // TerraformRepository is the Schema for the terraformrepositories API
 type TerraformRepository struct {

cmd/controllers/start.go

@@ -27,10 +27,12 @@ func buildControllersStartCmd(app *burrito.App) *cobra.Command {
 	defaultOnErrorTimer, _ := time.ParseDuration("10s")
 	defaultWaitActionTimer, _ := time.ParseDuration("5s")
 	defaultFailureGracePeriod, _ := time.ParseDuration("15s")
+	defaultRepositorySyncTimer, _ := time.ParseDuration("5m")
 
 	cmd.Flags().StringSliceVar(&app.Config.Controller.Namespaces, "namespaces", []string{"burrito-system"}, "list of namespaces to watch")
-	cmd.Flags().StringSliceVar(&app.Config.Controller.Types, "types", []string{"layer", "repository", "run", "pullrequest"}, "list of controllers to start")
+	cmd.Flags().StringArrayVar(&app.Config.Controller.Types, "types", []string{"layer", "run", "pullrequest"}, "list of controllers to start")
 	cmd.Flags().DurationVar(&app.Config.Controller.Timers.DriftDetection, "drift-detection-period", defaultDriftDetectionTimer, "period between two plans. Must end with s, m or h.")
+	cmd.Flags().DurationVar(&app.Config.Controller.Timers.RepositorySync, "repository-sync-period", defaultRepositorySyncTimer, "period between two repository sync. Must end with s, m or h.")
 	cmd.Flags().DurationVar(&app.Config.Controller.Timers.OnError, "on-error-period", defaultOnErrorTimer, "period between two runners launch when an error occurred in the controllers. Must end with s, m or h.")
 	cmd.Flags().DurationVar(&app.Config.Controller.Timers.WaitAction, "wait-action-period", defaultWaitActionTimer, "period between two runners when a layer is locked. Must end with s, m or h.")
 	cmd.Flags().DurationVar(&app.Config.Controller.Timers.FailureGracePeriod, "failure-grace-period", defaultFailureGracePeriod, "initial time before retry, goes exponential function of number failure. Must end with s, m or h.")

deploy/charts/burrito/templates/controllers.yaml

@@ -60,6 +60,10 @@ spec:
             - name: burrito-config
               mountPath: /etc/burrito
               readOnly: true
+            - name: ssh-known-hosts
+              mountPath: /home/burrito/.ssh/known_hosts
+              subPath: known_hosts
+              readOnly: true
             - name: burrito-token
               mountPath: /var/run/secrets/token
               readOnly: true
@@ -88,6 +92,9 @@ spec:
         - name: burrito-config
           configMap:
             name: burrito-config
+        - name: ssh-known-hosts
+          configMap:
+            name: burrito-ssh-known-hosts
         - name: burrito-token
           projected:
             sources:

deploy/charts/burrito/values.yaml

@@ -18,6 +18,8 @@ config:
       timers:
         # -- Drift detection interval
         driftDetection: 10m
+        # -- Repository polling interval
+        repositorySync: 5m
         # -- Duration to wait before retrying on error
         onError: 10s
         # -- Duration to wait before retrying on locked layer
@@ -28,8 +30,9 @@ config:
       maxConcurrentReconciles: 1
       # -- Maximum number of retries for Terraform operations (plan, apply...)
       terraformMaxRetries: 3
-      # -- Resource types to watch for reconciliation
-      types: ["layer", "repository", "run", "pullrequest"]
+      # TODO: enable repository controller by default
+      # -- Resource types to watch for reconciliation. Note: by default repository controller is disabled as it is not yet fully usable.
+      types: ["layer", "run", "pullrequest"]
       leaderElection:
         # -- Enable/Disable leader election
         enabled: true
@@ -271,7 +274,9 @@ controllers:
     # -- Environment variables to pass to the Burrito controller container
     envFrom: []
     # -- Environment variables to pass to the Burrito controller container
-    env: []
+    env:
+      - name: SSH_KNOWN_HOSTS
+        value: /home/burrito/.ssh/known_hosts
     # -- Additional volumes
     extraVolumes: {}
     # -- Additional volume mounts
@@ -314,7 +319,7 @@ server:
       initialDelaySeconds: 5
       periodSeconds: 20
     # -- Environment variables to pass to the Burrito server container
-    env: [] 
+    env: []
     # -- Environment variables to pass to the Burrito server container
     envFrom: []
     # -- Additional volumes

internal/annotations/annotations.go

@@ -2,6 +2,7 @@ package annotations
 
 import (
 	"context"
+	"strings"
 
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -21,13 +22,18 @@ const (
 	LastBranchCommitDate   string = "webhook.terraform.padok.cloud/branch-commit-date"
 	LastRelevantCommit     string = "webhook.terraform.padok.cloud/relevant-commit"
 	LastRelevantCommitDate string = "webhook.terraform.padok.cloud/relevant-commit-date"
+	SyncBranchNow          string = "webhook.terraform.padok.cloud/sync-"
 
 	ForceApply              string = "notifications.terraform.padok.cloud/force-apply"
 	AdditionnalTriggerPaths string = "config.terraform.padok.cloud/additionnal-trigger-paths"
 
 	SyncNow string = "api.terraform.padok.cloud/sync-now"
 )
 
+func ComputeKeyForSyncBranchNow(branch string) string {
+	return SyncBranchNow + strings.ReplaceAll(branch, "/", "--")
+}
+
 func Add(ctx context.Context, c client.Client, obj client.Object, annotations map[string]string) error {
 	newObj := obj.DeepCopyObject().(client.Object)
 	patch := client.MergeFrom(newObj)

internal/burrito/config/config.go

@@ -100,6 +100,7 @@ type ControllerTimers struct {
 	OnError            time.Duration `mapstructure:"onError"`
 	WaitAction         time.Duration `mapstructure:"waitAction"`
 	FailureGracePeriod time.Duration `mapstructure:"failureGracePeriod"`
+	RepositorySync     time.Duration `mapstructure:"repositorySync"`
 }
 
 type RepositoryConfig struct {
@@ -233,6 +234,7 @@ func TestConfig() *Config {
 				WaitAction:         5 * time.Minute,
 				FailureGracePeriod: 15 * time.Second,
 				OnError:            1 * time.Minute,
+				RepositorySync:     5 * time.Minute,
 			},
 		},
 		Runner: RunnerConfig{

internal/controllers/manager.go

@@ -108,6 +108,8 @@ func (c *Controllers) Exec() {
 		panic(err.Error())
 	}
 
+	log.Infof("starting these controllers: %v", c.config.Controller.Types)
+
 	for _, ctrlType := range c.config.Controller.Types {
 		switch ctrlType {
 		case "layer":
@@ -123,10 +125,11 @@ func (c *Controllers) Exec() {
 			log.Infof("layer controller started successfully")
 		case "repository":
 			if err = (&terraformrepository.Reconciler{
-				Client:   mgr.GetClient(),
-				Scheme:   mgr.GetScheme(),
-				Recorder: mgr.GetEventRecorderFor("Burrito"),
-				Config:   c.config,
+				Client:    mgr.GetClient(),
+				Scheme:    mgr.GetScheme(),
+				Recorder:  mgr.GetEventRecorderFor("Burrito"),
+				Config:    c.config,
+				Datastore: datastoreClient,
 			}).SetupWithManager(mgr); err != nil {
 				log.Fatalf("unable to create repository controller: %s", err)
 			}

internal/controllers/terraformpullrequest/controller.go

@@ -3,7 +3,6 @@ package terraformpullrequest
 import (
 	"context"
 	"fmt"
-	"strconv"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/padok-team/burrito/internal/burrito/config"
@@ -24,6 +23,7 @@ import (
 	configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1"
 	"github.com/padok-team/burrito/internal/utils/gitprovider"
 	gt "github.com/padok-team/burrito/internal/utils/gitprovider/types"
+	"github.com/padok-team/burrito/internal/utils/typeutils"
 )
 
 // Reconciler reconciles a TerraformPullRequest object
@@ -147,7 +147,6 @@ func (r *Reconciler) initializeProvider(ctx context.Context, repository *configv
 		log.Debugf("no secret configured for repository %s/%s, skipping provider initialization", repository.Namespace, repository.Name)
 		return nil, nil
 	}
-	log.Infof("KUBE API REQUEST: getting secret %s/%s", repository.Namespace, repository.Spec.Repository.SecretName)
 	secret := &corev1.Secret{}
 	err := r.Client.Get(ctx, types.NamespacedName{
 		Name:      repository.Spec.Repository.SecretName,
@@ -158,9 +157,9 @@ func (r *Reconciler) initializeProvider(ctx context.Context, repository *configv
 		return nil, err
 	}
 	config := gitprovider.Config{
-		AppID:             parseSecretInt64(secret.Data["githubAppId"]),
+		AppID:             typeutils.ParseSecretInt64(secret.Data["githubAppId"]),
 		URL:               repository.Spec.Repository.Url,
-		AppInstallationID: parseSecretInt64(secret.Data["githubAppInstallationId"]),
+		AppInstallationID: typeutils.ParseSecretInt64(secret.Data["githubAppInstallationId"]),
 		AppPrivateKey:     string(secret.Data["githubAppPrivateKey"]),
 		GitHubToken:       string(secret.Data["githubToken"]),
 		GitLabToken:       string(secret.Data["gitlabToken"]),
@@ -205,8 +204,3 @@ func (r *Reconciler) initializeDefaultProviders() error {
 	}
 	return nil
 }
-
-func parseSecretInt64(data []byte) int64 {
-	v, _ := strconv.ParseInt(string(data), 10, 64)
-	return v
-}