Merge pull request #308 from vishnoianil/delete-keys

Cleanup redis keys once job results are posted.

gobot/cmd/root.go

@@ -16,6 +16,7 @@ import (
 	gosmee "github.com/chmouel/gosmee/gosmee"
 	"github.com/go-redis/redis"
 	"github.com/gregjones/httpcache"
+	"github.com/instructlab/instructlab-bot/gobot/common"
 	"github.com/instructlab/instructlab-bot/gobot/handlers"
 	"github.com/instructlab/instructlab-bot/gobot/util"
 	"github.com/palantir/go-githubapp/githubapp"
@@ -257,13 +258,13 @@ func receiveResults(ctx context.Context, redisHostPort string, logger *zap.Sugar
 			if result == "" {
 				continue
 			}
-			prNumber, err := r.Get(buildRedisKey(result, handlers.RedisKeyPRNumber)).Result()
+			prNumber, err := r.Get(buildRedisKey(result, common.RedisKeyPRNumber)).Result()
 			if err != nil || prNumber == "" {
 				logger.Errorf("No PR number found for job %s", result)
 				continue
 			}
 
-			installID, err := r.Get(buildRedisKey(result, handlers.RedisKeyInstallationID)).Result()
+			installID, err := r.Get(buildRedisKey(result, common.RedisKeyInstallationID)).Result()
 			if err != nil || installID == "" {
 				logger.Errorf("No installation ID found for job %s", result)
 				continue
@@ -274,31 +275,31 @@ func receiveResults(ctx context.Context, redisHostPort string, logger *zap.Sugar
 				continue
 			}
 
-			repoOwner, err := r.Get(buildRedisKey(result, handlers.RedisKeyRepoOwner)).Result()
+			repoOwner, err := r.Get(buildRedisKey(result, common.RedisKeyRepoOwner)).Result()
 			if err != nil || repoOwner == "" {
 				logger.Errorf("No repo owner found for job %s", result)
 				continue
 			}
 
-			repoName, err := r.Get(buildRedisKey(result, handlers.RedisKeyRepoName)).Result()
+			repoName, err := r.Get(buildRedisKey(result, common.RedisKeyRepoName)).Result()
 			if err != nil || repoName == "" {
 				logger.Errorf("No repo name found for job %s", result)
 				continue
 			}
 
-			jobType, err := r.Get(buildRedisKey(result, handlers.RedisKeyJobType)).Result()
+			jobType, err := r.Get(buildRedisKey(result, common.RedisKeyJobType)).Result()
 			if err != nil || jobType == "" {
 				logger.Errorf("No job type found for job %s", result)
 				continue
 			}
 
-			prSha, err := r.Get(buildRedisKey(result, handlers.RedisKeyPRSHA)).Result()
+			prSha, err := r.Get(buildRedisKey(result, common.RedisKeyPRSHA)).Result()
 			if err != nil || prSha == "" {
 				logger.Errorf("No PR SHA found for job %s", result)
 				continue
 			}
 
-			requestTimeStr, err := r.Get(buildRedisKey(result, handlers.RedisKeyRequestTime)).Result()
+			requestTimeStr, err := r.Get(buildRedisKey(result, common.RedisKeyRequestTime)).Result()
 			if err != nil || requestTimeStr == "" {
 				logger.Errorf("No request time found for job %s", result)
 				continue
@@ -312,7 +313,7 @@ func receiveResults(ctx context.Context, redisHostPort string, logger *zap.Sugar
 
 			prURL := fmt.Sprintf("https://github.com/%s/%s/pull/%s", repoOwner, repoName, prNumber)
 
-			jobDuration, err := r.Get(buildRedisKey(result, handlers.RedisKeyDuration)).Result()
+			jobDuration, err := r.Get(buildRedisKey(result, common.RedisKeyDuration)).Result()
 			if err != nil || jobDuration == "" {
 				logger.Infof("Job result for %s/%s#%s, job ID: %s, GitHub URL: %s (No job duration time found for job)", repoOwner, repoName, prNumber, result, prURL)
 			} else {
@@ -329,11 +330,11 @@ func receiveResults(ctx context.Context, redisHostPort string, logger *zap.Sugar
 			var statusContext string
 			switch jobType {
 			case "generate":
-				statusContext = util.GenerateLocalCheck
+				statusContext = common.GenerateLocalCheck
 			case "precheck":
-				statusContext = util.PrecheckCheck
+				statusContext = common.PrecheckCheck
 			case "sdg-svc":
-				statusContext = util.GenerateSDGCheck
+				statusContext = common.GenerateSDGCheck
 			default:
 				logger.Errorf("Unknown job type: %s", jobType)
 			}
@@ -356,8 +357,8 @@ func receiveResults(ctx context.Context, redisHostPort string, logger *zap.Sugar
 				errCommentBody := fmt.Sprintf("An error occurred while processing your request, please review the following log for job id %s :\n\n```\n%s\n```", result, prErrors)
 
 				params := util.PullRequestStatusParams{
-					Status:       util.CheckComplete,
-					Conclusion:   util.CheckStatusFailure,
+					Status:       common.CheckComplete,
+					Conclusion:   common.CheckStatusFailure,
 					CheckName:    statusContext,
 					CheckSummary: JobFailed,
 					CheckDetails: errCommentBody,
@@ -378,6 +379,9 @@ func receiveResults(ctx context.Context, redisHostPort string, logger *zap.Sugar
 				if err != nil {
 					logger.Errorf("Failed to update error message on PR for job %s error: %v", result, err)
 				}
+
+				// Enable redis keys deletion once we have solution for persisting the job history
+				// cleanupRedisKeys(logger, r, result)
 				continue
 			}
 
@@ -405,8 +409,8 @@ func receiveResults(ctx context.Context, redisHostPort string, logger *zap.Sugar
 			summaryMsg := fmt.Sprintf("Job ID: %s completed successfully. Check Details.", result)
 
 			params := util.PullRequestStatusParams{
-				Status:       util.CheckComplete,
-				Conclusion:   util.CheckStatusSuccess,
+				Status:       common.CheckComplete,
+				Conclusion:   common.CheckStatusSuccess,
 				JobID:        result,
 				JobType:      jobType,
 				CheckName:    statusContext,
@@ -428,11 +432,40 @@ func receiveResults(ctx context.Context, redisHostPort string, logger *zap.Sugar
 			if err != nil {
 				logger.Errorf("Failed to post comment on pr %s/%s#%d: %v", params.RepoOwner, params.RepoName, params.PrNum, err)
 			}
+			// Enable redis keys deletion once we have solution for persisting the job history
+			// cleanupRedisKeys(logger, r, result)
 		}
 	}
 }
 
 // buildRedisKey constructs a Redis key for job attributes.
 func buildRedisKey(jobID, keyType string) string {
-	return fmt.Sprintf("%s:%s:%s", handlers.RedisKeyJobs, jobID, keyType)
+	return fmt.Sprintf("%s:%s:%s", common.RedisKeyJobs, jobID, keyType)
+}
+
+//lint:ignore U1000
+func cleanupRedisKeys(logger *zap.SugaredLogger, r *redis.Client, jobID string) {
+	matchKey := fmt.Sprintf("%s:%s:*", common.RedisKeyJobs, jobID)
+	var cursor uint64 = 0
+
+	for {
+		keys, nextCursor, err := r.Scan(cursor, matchKey, 0).Result()
+		if err != nil {
+			logger.Errorf("Error scanning keys matching to %s: %v", matchKey, err)
+			return
+		}
+
+		for _, key := range keys {
+			err := r.Del(key).Err()
+			if err != nil {
+				logger.Warnf("Error deleting key %s: %v", key, err)
+			}
+		}
+
+		cursor = nextCursor
+		if cursor == 0 {
+			break
+		}
+	}
+	logger.Infof("Deleted all keys matching to %s", matchKey)
 }

gobot/common/constants.go

@@ -0,0 +1,41 @@
+package common
+
+const (
+	RepoName = "taxonomy"
+
+	CheckComplete      = "completed"
+	CheckQueued        = "queued"
+	CheckInProgress    = "in_progress"
+	CheckStatusSuccess = "success"
+	CheckStatusFailure = "failure"
+	CheckStatusError   = "error"
+	CheckStatusPending = "pending"
+
+	BotReadyStatus    = "InstructLab Bot"
+	BotReadyStatusMsg = "InstructLab bot is ready to assist!!"
+
+	PrecheckCheck      = "Precheck Check"
+	GenerateLocalCheck = "Generate Local Check"
+	GenerateSDGCheck   = "Generate SDG Check"
+
+	PrecheckStatus      = "Precheck Status"
+	GenerateLocalStatus = "Generate Local Status"
+	GenerateSDGStatus   = "Generate SDG Status"
+
+	InstructLabBotUrl = "https://github.com/instructlab/instructlab-bot"
+)
+
+const (
+	RedisKeyJobs           = "jobs"
+	RedisKeyPRNumber       = "pr_number"
+	RedisKeyPRSHA          = "pr_sha"
+	RedisKeyAuthor         = "author"
+	RedisKeyInstallationID = "installation_id"
+	RedisKeyRepoOwner      = "repo_owner"
+	RedisKeyRepoName       = "repo_name"
+	RedisKeyJobType        = "job_type"
+	RedisKeyErrors         = "errors"
+	RedisKeyRequestTime    = "request_time"
+	RedisKeyDuration       = "duration"
+	RedisKeyStatus         = "status"
+)

gobot/handlers/issue_comment.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/go-redis/redis/v8"
 	"github.com/google/go-github/v60/github"
+	"github.com/instructlab/instructlab-bot/gobot/common"
 	"github.com/instructlab/instructlab-bot/gobot/util"
 	"github.com/palantir/go-githubapp/githubapp"
 	"github.com/pkg/errors"
@@ -24,21 +25,6 @@ const (
 	BotEnabled        = "Bot is successfully enabled."
 )
 
-const (
-	RedisKeyJobs           = "jobs"
-	RedisKeyPRNumber       = "pr_number"
-	RedisKeyPRSHA          = "pr_sha"
-	RedisKeyAuthor         = "author"
-	RedisKeyInstallationID = "installation_id"
-	RedisKeyRepoOwner      = "repo_owner"
-	RedisKeyRepoName       = "repo_name"
-	RedisKeyJobType        = "job_type"
-	RedisKeyErrors         = "errors"
-	RedisKeyRequestTime    = "request_time"
-	RedisKeyDuration       = "duration"
-	RedisKeyStatus         = "status"
-)
-
 type PRCommentHandler struct {
 	githubapp.ClientCreator
 	Logger         *zap.SugaredLogger
@@ -71,6 +57,12 @@ func (h *PRCommentHandler) Handle(ctx context.Context, eventType, deliveryID str
 		return errors.Wrap(err, "failed to parse issue comment event payload")
 	}
 
+	if event.GetRepo().GetName() != common.RepoName {
+		h.Logger.Warnf("Received unexpected event %s from %s/%s repo. Skipping the event.",
+			eventType, event.GetOrganization().GetLogin(), event.GetRepo().GetName())
+		return nil
+	}
+
 	if !event.GetIssue().IsPullRequest() {
 		return nil
 	}
@@ -152,57 +144,57 @@ func (h *PRCommentHandler) queueGenerateJob(ctx context.Context, client *github.
 		DB:       0,  // use default DB
 	})
 
-	jobNumber, err := r.Incr(ctx, RedisKeyJobs).Result()
+	jobNumber, err := r.Incr(ctx, common.RedisKeyJobs).Result()
 	if err != nil {
 		return err
 	}
 
-	err = setJobKey(r, jobNumber, RedisKeyPRNumber, prComment.prNum)
+	err = setJobKey(r, jobNumber, common.RedisKeyPRNumber, prComment.prNum)
 	if err != nil {
 		return err
 	}
 
-	err = setJobKey(r, jobNumber, RedisKeyPRSHA, prComment.prSha)
+	err = setJobKey(r, jobNumber, common.RedisKeyPRSHA, prComment.prSha)
 	if err != nil {
 		return err
 	}
 
-	err = setJobKey(r, jobNumber, RedisKeyAuthor, prComment.author)
+	err = setJobKey(r, jobNumber, common.RedisKeyAuthor, prComment.author)
 	if err != nil {
 		return err
 	}
 
-	err = setJobKey(r, jobNumber, RedisKeyInstallationID, prComment.installID)
+	err = setJobKey(r, jobNumber, common.RedisKeyInstallationID, prComment.installID)
 	if err != nil {
 		return err
 	}
 
-	err = setJobKey(r, jobNumber, RedisKeyRepoOwner, prComment.repoOwner)
+	err = setJobKey(r, jobNumber, common.RedisKeyRepoOwner, prComment.repoOwner)
 	if err != nil {
 		return err
 	}
 
-	err = setJobKey(r, jobNumber, RedisKeyRepoName, prComment.repoName)
+	err = setJobKey(r, jobNumber, common.RedisKeyRepoName, prComment.repoName)
 	if err != nil {
 		return err
 	}
 
-	err = setJobKey(r, jobNumber, RedisKeyJobType, jobType)
+	err = setJobKey(r, jobNumber, common.RedisKeyJobType, jobType)
 	if err != nil {
 		return err
 	}
 
-	err = setJobKey(r, jobNumber, RedisKeyErrors, "")
+	err = setJobKey(r, jobNumber, common.RedisKeyErrors, "")
 	if err != nil {
 		return err
 	}
 
-	err = setJobKey(r, jobNumber, RedisKeyStatus, util.CheckStatusPending)
+	err = setJobKey(r, jobNumber, common.RedisKeyStatus, common.CheckStatusPending)
 	if err != nil {
 		return err
 	}
 
-	err = setJobKey(r, jobNumber, RedisKeyRequestTime, strconv.FormatInt(time.Now().Unix(), 10))
+	err = setJobKey(r, jobNumber, common.RedisKeyRequestTime, strconv.FormatInt(time.Now().Unix(), 10))
 	if err != nil {
 		return err
 	}
@@ -225,20 +217,20 @@ func (h *PRCommentHandler) queueGenerateJob(ctx context.Context, client *github.
 	var statusName string
 	switch jobType {
 	case "generate":
-		checkName = util.GenerateLocalCheck
-		statusName = util.GenerateLocalStatus
+		checkName = common.GenerateLocalCheck
+		statusName = common.GenerateLocalStatus
 	case "precheck":
-		checkName = util.PrecheckCheck
-		statusName = util.PrecheckStatus
+		checkName = common.PrecheckCheck
+		statusName = common.PrecheckStatus
 	case "sdg-svc":
-		checkName = util.GenerateSDGCheck
-		statusName = util.GenerateSDGStatus
+		checkName = common.GenerateSDGCheck
+		statusName = common.GenerateSDGStatus
 	default:
 		h.Logger.Errorf("Unknown job type: %s", jobType)
 	}
 
 	params := util.PullRequestStatusParams{
-		Status:       util.CheckInProgress,
+		Status:       common.CheckInProgress,
 		CheckSummary: summaryMsg,
 		CheckDetails: detailsMsg,
 		Comment:      commentMsg,
@@ -340,9 +332,9 @@ func (h *PRCommentHandler) generateCommand(ctx context.Context, client *github.C
 		prComment.repoOwner, prComment.repoName, prComment.prNum, prComment.author)
 
 	params := util.PullRequestStatusParams{
-		Status:     util.CheckComplete,
-		Conclusion: util.CheckStatusFailure,
-		CheckName:  util.GenerateLocalCheck,
+		Status:     common.CheckComplete,
+		Conclusion: common.CheckStatusFailure,
+		CheckName:  common.GenerateLocalCheck,
 		RepoOwner:  prComment.repoOwner,
 		RepoName:   prComment.repoName,
 		PrNum:      prComment.prNum,
@@ -387,9 +379,9 @@ func (h *PRCommentHandler) precheckCommand(ctx context.Context, client *github.C
 		prComment.repoOwner, prComment.repoName, prComment.prNum, prComment.author)
 
 	params := util.PullRequestStatusParams{
-		Status:     util.CheckComplete,
-		Conclusion: util.CheckStatusFailure,
-		CheckName:  util.PrecheckCheck,
+		Status:     common.CheckComplete,
+		Conclusion: common.CheckStatusFailure,
+		CheckName:  common.PrecheckCheck,
 		RepoOwner:  prComment.repoOwner,
 		RepoName:   prComment.repoName,
 		PrNum:      prComment.prNum,
@@ -433,9 +425,9 @@ func (h *PRCommentHandler) sdgSvcCommand(ctx context.Context, client *github.Cli
 		prComment.repoOwner, prComment.repoName, prComment.prNum, prComment.author)
 
 	params := util.PullRequestStatusParams{
-		Status:     util.CheckComplete,
-		Conclusion: util.CheckStatusFailure,
-		CheckName:  util.GenerateSDGCheck,
+		Status:     common.CheckComplete,
+		Conclusion: common.CheckStatusFailure,
+		CheckName:  common.GenerateSDGCheck,
 		RepoOwner:  prComment.repoOwner,
 		RepoName:   prComment.repoName,
 		PrNum:      prComment.prNum,