[PM-11376] Improve automated commit message (#29)

bitwarden · Sep 11, 2024 · d0b9b3c · d0b9b3c
1 parent 3ac3386
commit d0b9b3c
Show file tree

Hide file tree

Showing 5 changed files with 260 additions and 173 deletions.
diff --git a/.github/workflows/fdroid.yml b/.github/workflows/fdroid.yml
@@ -19,7 +19,8 @@ jobs:
   apps:
     name: "Generate repo from apps listing"
     runs-on: ubuntu-22.04
-
+    env:
+      COMMIT_MSG_FILE: "${{ github.workspace }}/commit_message.tmp"
 
     steps:
       - name: Checkout repo
@@ -100,10 +101,16 @@ jobs:
         with:
           go-version: '^1.17.0'
 
-      - name: Run update script
+      - name: Run metascoop
+        id: run-metascoop
         env:
           GH_ACCESS_TOKEN: ${{ steps.retrieve-secrets.outputs.github-pat-bitwarden-devops-bot-repo-scope }}
+        run: |
+          bash run_metascoop.sh ${{ env.COMMIT_MSG_FILE }}
+
+      - name: Update repo
+        env:
           GH_TOKEN: ${{ steps.retrieve-secrets.outputs.github-pat-bitwarden-devops-bot-repo-scope }}
-          DRY_RUN: ${{ !contains(inputs.dry-run, 'true') }}
+        if: ${{ (github.event_name == 'schedule' || inputs.dry-run == 'false') }}
         run: |
-          bash update.sh --dry-run ${{ inputs.dry-run }}
+          bash update_repo.sh ${{ env.COMMIT_MSG_FILE }}
diff --git a/metascoop/main.go b/metascoop/main.go
@@ -13,6 +13,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"sync"
 	"time"
 
 	"metascoop/apps"
@@ -26,11 +27,11 @@ import (
 
 func main() {
 	var (
-		reposFilePath = flag.String("rp", "repos.yaml", "Path to repos.yaml file")
-		repoDir       = flag.String("rd", "fdroid/repo", "Path to fdroid \"repo\" directory")
-		accessToken   = flag.String("pat", "", "GitHub personal access token")
-
-		debugMode = flag.Bool("debug", false, "Debug mode won't run the fdroid command")
+		reposFilePath  = flag.String("rp", "repos.yaml", "Path to repos.yaml file")
+		repoDir        = flag.String("rd", "fdroid/repo", "Path to fdroid \"repo\" directory")
+		accessToken    = flag.String("pat", "", "GitHub personal access token")
+		commitMsgFile  = flag.String("cm", "commit_message.tmp", "Path to the commit message file")
+		debugMode      = flag.Bool("debug", false, "Debug mode won't run the fdroid command")
 	)
 	flag.Parse()
 
@@ -69,112 +70,196 @@ func main() {
 		haveError          bool
 		apkInfoMap         = make(map[string]apps.Application)
 		toRemovePaths      []string
+		changedRepos       = make(map[string]map[string]*github.RepositoryRelease)
+		mu                 sync.Mutex
+		wg                 sync.WaitGroup
 	)
 
-	// Track if changes are detected that will require re-generating metadata
 	regenerateMetadata := false
-	for _, repo := range reposList {
-		fmt.Printf("::group::Repo: %s/%s\n", repo.Owner, repo.Name)
-
-		err, releases := getRepositoryReleases(githubClient, repo)
-		if err != nil {
-			log.Printf("Error while listing repo releases for %q: %s\n", repo.GitURL, err.Error())
-			haveError = true
-			return
-		}
-
-		log.Printf("Received %d releases", len(releases))
-
-		for _, app := range repo.Applications {
-			fmt.Printf("::group::App %s\n", app.Name)
 
-			foundArtifact := false
-
-			for _, release := range releases {
-				fmt.Printf("::group::Release %s\n", release.GetTagName())
-
-				if release.GetDraft() {
-					log.Printf("Skipping draft %q\n", release.GetTagName())
-					continue
-				}
-				if release.GetTagName() == "" {
-					log.Printf("Skipping release with empty tag name")
-					continue
-				}
-
-				log.Printf("Working on release with tag name %q", release.GetTagName())
-				var apk *github.ReleaseAsset = apps.FindAPK(release, app.Filename)
-
-				if apk == nil {
-					log.Printf("Couldn't find any F-Droid assets for application %s in %s with file name %s", app.Filename, release.GetName(), app.Filename)
-					continue
-				}
-
-				appName := apps.GenerateReleaseFilename(app.Id, release.GetTagName())
-
-				log.Printf("Target APK name: %s\n", appName)
-
-				appClone := app
+	for _, repo := range reposList {
+		wg.Add(1)
+		go func(repo apps.Repo) {
+			defer wg.Done()
 
-				appClone.ReleaseDescription = release.GetBody()
-				if appClone.ReleaseDescription != "" {
-					log.Printf("Release notes: \n%s\n", appClone.ReleaseDescription)
-				}
+			fmt.Printf("::group::Repo: %s/%s\n", repo.Owner, repo.Name)
 
-				apkInfoMap[appName] = appClone
+			err, releases := getRepositoryReleases(githubClient, repo)
+			if err != nil {
+				log.Printf("Error while listing repo releases for %q: %s\n", repo.GitURL, err.Error())
+				mu.Lock()
+				haveError = true
+				mu.Unlock()
+				return
+			}
 
-				appTargetPath := filepath.Join(*repoDir, appName)
+			log.Printf("Received %d releases", len(releases))
+
+			var appWg sync.WaitGroup
+			repoChanged := false
+			for _, app := range repo.Applications {
+				appWg.Add(1)
+				go func(app apps.Application) {
+					defer appWg.Done()
+
+					fmt.Printf("::group::App %s\n", app.Name)
+
+					foundArtifact := false
+
+					for _, release := range releases {
+						fmt.Printf("::group::Release %s\n", release.GetTagName())
+
+						if release.GetDraft() {
+							log.Printf("Skipping draft %q\n", release.GetTagName())
+							continue
+						}
+						if release.GetTagName() == "" {
+							log.Printf("Skipping release with empty tag name")
+							continue
+						}
+
+						log.Printf("Working on release with tag name %q", release.GetTagName())
+						var apk *github.ReleaseAsset = apps.FindAPK(release, app.Filename)
+
+						if apk == nil {
+							log.Printf("Couldn't find any F-Droid assets for application %s in %s with file name %s", app.Filename, release.GetName(), app.Filename)
+							continue
+						}
+
+						appName := apps.GenerateReleaseFilename(app.Id, release.GetTagName())
+
+						log.Printf("Target APK name: %s\n", appName)
+
+						appClone := app
+
+						appClone.ReleaseDescription = release.GetBody()
+						if appClone.ReleaseDescription != "" {
+							log.Printf("Release notes: \n%s\n", appClone.ReleaseDescription)
+						}
+
+						mu.Lock()
+						apkInfoMap[appName] = appClone
+						mu.Unlock()
+
+						appTargetPath := filepath.Join(*repoDir, appName)
+
+						// If the app file already exists for this version, we stop processing this app and move to the next
+						if _, err := os.Stat(appTargetPath); !errors.Is(err, os.ErrNotExist) {
+							log.Printf("Already have APK for version %q at %q\n", release.GetTagName(), appTargetPath)
+							foundArtifact = true
+							break
+						}
+
+						log.Printf("Downloading APK %q from release %q to %q", apk.GetName(), release.GetTagName(), appTargetPath)
+
+						downloadContext, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+						defer cancel()
+
+						appStream, _, err := githubClient.Repositories.DownloadReleaseAsset(downloadContext, repo.Owner, repo.Name, apk.GetID(), http.DefaultClient)
+						if err != nil {
+							log.Printf("Error while downloading app %q (artifact id %d) from from release %q: %s", repo.GitURL, apk.GetID(), release.GetTagName(), err.Error())
+							mu.Lock()
+							haveError = true
+							mu.Unlock()
+							break
+						}
+
+						err = downloadStream(appTargetPath, appStream)
+						if err != nil {
+							log.Printf("Error while downloading app %q (artifact id %d) from from release %q to %q: %s", repo.GitURL, *apk.ID, *release.TagName, appTargetPath, err.Error())
+							mu.Lock()
+							haveError = true
+							mu.Unlock()
+							break
+						}
+
+						log.Printf("Successfully downloaded app for version %q", release.GetTagName())
+						fmt.Printf("::endgroup:App %s\n", app.Name)
+						mu.Lock()
+						regenerateMetadata = true
+						repoChanged = true
+						if changedRepos[repo.GitURL] == nil {
+							changedRepos[repo.GitURL] = make(map[string]*github.RepositoryRelease)
+						}
+						changedRepos[repo.GitURL][app.Filename] = release
+						mu.Unlock()
+						break
+					}
+					if foundArtifact || haveError {
+						// Stop after the first [release] of this [app] is downloaded to prevent back-filling legacy releases.
+						return
+					}
+				}(app)
+			}
 
-				// If the app file already exists for this version, we stop processing this app and move to the next
-				if _, err := os.Stat(appTargetPath); !errors.Is(err, os.ErrNotExist) {
-					log.Printf("Already have APK for version %q at %q\n", release.GetTagName(), appTargetPath)
-					foundArtifact = true
-					break
-				}
+			appWg.Wait()
 
-				log.Printf("Downloading APK %q from release %q to %q", apk.GetName(), release.GetTagName(), appTargetPath)
+			if repoChanged {
+				log.Printf("Changes detected for repo: %s", repo.GitURL)
+			}
 
-				downloadContext, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
-				defer cancel()
+			fmt.Printf("::endgroup::Repo: %s/%s\n", repo.Owner, repo.Name)
+		}(repo)
+	}
 
-				appStream, _, err := githubClient.Repositories.DownloadReleaseAsset(downloadContext, repo.Owner, repo.Name, apk.GetID(), http.DefaultClient)
-				if err != nil {
-					log.Printf("Error while downloading app %q (artifact id %d) from from release %q: %s", repo.GitURL, apk.GetID(), release.GetTagName(), err.Error())
-					haveError = true
-					break
+	wg.Wait()
+
+	// Write changes to temporary commit message file
+	if regenerateMetadata {
+		var commitMsg strings.Builder
+
+		// Create the first line with repo names
+		repoNames := make([]string, 0, len(changedRepos))
+		for repoURL := range changedRepos {
+			repoName := strings.TrimPrefix(repoURL, "https://github.com/")
+			repoNames = append(repoNames, repoName)
+		}
+		commitMsg.WriteString(fmt.Sprintf("Updated apps from %s\n\n", strings.Join(repoNames, ", ")))
+
+		// Add details for each repo
+		for repoURL, apps := range changedRepos {
+			repoFullName := strings.TrimPrefix(repoURL, "https://github.com/")
+
+			commitMsg.WriteString(fmt.Sprintf("<details>\n<summary>%s</summary>\n\n", repoFullName))
+
+			// Group apps by release
+			releaseApps := make(map[*github.RepositoryRelease][]string)
+			for appFilename, release := range apps {
+				releaseApps[release] = append(releaseApps[release], appFilename)
+			}
+
+			for release, appList := range releaseApps {
+				releaseName := release.GetName()
+				if releaseName == "" {
+					releaseName = release.GetTagName()
 				}
-
-				err = downloadStream(appTargetPath, appStream)
-				if err != nil {
-					log.Printf("Error while downloading app %q (artifact id %d) from from release %q to %q: %s", repo.GitURL, *apk.ID, *release.TagName, appTargetPath, err.Error())
-					haveError = true
-					break
+				releaseTagURL := release.GetHTMLURL()
+				commitMsg.WriteString(fmt.Sprintf("### [%s](%s)\n\n", releaseName, releaseTagURL))
+
+				for _, appFilename := range appList {
+					commitMsg.WriteString(fmt.Sprintf("- %s\n", appFilename))
 				}
-
-				log.Printf("Successfully downloaded app for version %q", release.GetTagName())
-				fmt.Printf("::endgroup:App %s\n", app.Name)
-				regenerateMetadata = true
-				break
-
-			}
-			if foundArtifact || haveError {
-				// Stop after the first [release] of this [app] is downloaded to prevent back-filling legacy releases.
-				break
+				commitMsg.WriteString("\n")
 			}
+
+			commitMsg.WriteString("</details>\n\n")
 		}
 
-		fmt.Printf("::endgroup::Repo: %s/%s\n", repo.Owner, repo.Name)
+		err := os.WriteFile(*commitMsgFile, []byte(commitMsg.String()), 0644)
+		if err != nil {
+			log.Printf("Error writing commit message file: %s", err)
+		} else {
+			log.Printf("Commit message written to %s", *commitMsgFile)
+		}
+	} else {
+		log.Printf("No changes detected.")
+		os.Exit(2)
 	}
 
 	if haveError {
 		os.Exit(1)
 	}
 
-	if !regenerateMetadata {
-		log.Printf("No changes detected.")
-		os.Exit(2)
-	}
-
 	if !*debugMode {
 		fmt.Println("::group::F-Droid: Creating metadata stubs")
 		// Now, we run the fdroid update command

diff --git a/run_metascoop.sh b/run_metascoop.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Check if the commit message file argument is provided
+if [ "$#" -ne 1 ]; then
+    echo "Usage: $0 <commit_message_file_path>"
+    exit 1
+fi
+
+COMMIT_MSG_FILE="$1"
+
+echo "::group::Building metascoop executable"
+cd metascoop
+go build -o metascoop
+echo "::endgroup::"
+
+echo "::group::Running metascoop"
+./metascoop -rp=../repos.yaml -rd=../fdroid/repo -pat="$GH_ACCESS_TOKEN" -cm="$COMMIT_MSG_FILE"
+EXIT_CODE=$?
+cd ..
+echo "::endgroup::"
+
+echo "Metascoop had an exit code of $EXIT_CODE"
+
+if [ $EXIT_CODE -eq 2 ]; then
+    echo "There were no significant changes"
+    exit 0
+elif [ $EXIT_CODE -eq 0 ]; then
+    echo "Changes detected"
+    exit 0
+else
+    echo "This is an unexpected error"
+    exit 1
+fi