wip first pass at test reports

cosmos · Jan 22, 2025 · 49c136f · 49c136f
1 parent 26fec1c
commit 49c136f
Show file tree

Hide file tree

Showing 3 changed files with 45 additions and 9 deletions.
diff --git a/.github/workflows/interchain-test.yml b/.github/workflows/interchain-test.yml
@@ -18,7 +18,7 @@ jobs:
                   echo "tag_name=${{ github.event.client_payload.tag_name }}" | tee -a $GITHUB_OUTPUT
                 else
                   echo "ref_name=${{ github.ref_name }}" | tee -a $GITHUB_OUTPUT
-                  echo "tag_name=${{ github.ref_name }}" | tee -a $GITHUB_OUTPUT
+                  echo "tag_name=${{ github.ref_name }}" | sed 's~/~-~g' | tee -a $GITHUB_OUTPUT
                 fi
             - name: Check out repository code
               uses: actions/checkout@v4
@@ -57,8 +57,41 @@ jobs:
                 TEST_NEW_GAIA_IMAGE_VERSION: "${{ matrix.test_version }}"
                 TEST_UPGRADE_NAME: "${{ matrix.upgrade_name }}"
               run: |
-                # This docker pull/tag is a quick hack only necessary for v19, since there were no official v18 images built.
-                # Once we're testing 19 -> 20 this can be removed
-                docker pull "ghcr.io/hyphacoop/gaia:v18.1.0" && docker tag "ghcr.io/hyphacoop/gaia:v18.1.0" "ghcr.io/${{ github.repository_owner }}/gaia:v18.1.0"
                 cd ./tests/interchain
-                go test -v ./... -failfast -p 1 -timeout 5h -run="^${{ matrix.test_name }}"
+                go install github.com/mfridman/tparse@latest
+                go test -v ./... -failfast -p 1 -timeout 5h -run="^${{ matrix.test_name }}" -json | tee ../../output-${{ matrix.previous_version }}-${{ matrix.test_name }}.json | tparse -follow -all
+            - name: Upload output
+              uses: actions/upload-artifact@v4
+              if: always()
+              with:
+                name: output-${{ matrix.previous_version }}-${{ matrix.test_name }}
+                path: output-${{ matrix.previous_version }}-${{ matrix.test_name }}.json
+    test-report:
+        needs: [test, prepare-matrix]
+        if: always()
+        runs-on: ubuntu-latest
+        steps:
+            - name: Download all artifacts
+              uses: actions/download-artifact@v4
+              with:
+                path: ./outputs
+            - name: Setup go
+              uses: actions/setup-go@v5
+            - name: Prep report
+              env:
+                TEST_MATRIX: ${{ needs.prepare-matrix.outputs.matrix }}
+              run: |
+                go install github.com/becheran/go-testreport@latest
+                TEST_VERSION=$(echo "$TEST_MATRIX" | jq -r '.test_version[0]')
+                UPGRADE_NAME=$(echo "$TEST_MATRIX" | jq -r '.upgrade_name[0]')
+                echo "$TEST_MATRIX" | jq -r '.previous_version[]' | while read PREV_VERSION; do
+                  cat ./outputs/output-${PREV_VERSION}-*/*.json > combined-${PREV_VERSION}.json
+                  go-testreport -vars="Title:${PREV_VERSION} -> ${UPGRADE_NAME} (${TEST_VERSION})" "test-report-${PREV_VERSION}.md" < "combined-${PREV_VERSION}.json"
+                done
+                cat test-report-*.md > test-report.md
+                cat test-report.md > $GITHUB_STEP_SUMMARY
+            - name: Upload output
+              uses: actions/upload-artifact@v4
+              with:
+                name: test-report
+                path: test-report.md
diff --git a/tests/interchain/chainsuite/suite.go b/tests/interchain/chainsuite/suite.go
@@ -47,6 +47,7 @@ func (s *Suite) SetupTest() {
 }
 
 func (s *Suite) SetupSuite() {
+	panic("Cut to the chase") // TODO: Remove this!
 	if s.Config.Scope == ChainScopeSuite {
 		s.createChain()
 	}

diff --git a/tests/interchain/matrix_tool/main.go b/tests/interchain/matrix_tool/main.go
@@ -29,7 +29,7 @@ func GetPreviousMajorMinor(ctx context.Context, testVersion string) (previousVer
 		org = "cosmos"
 	}
 	client := github.NewClient(nil)
-	releaes, _, err := client.Repositories.ListReleases(ctx, org, "gaia", nil)
+	releases, _, err := client.Repositories.ListReleases(ctx, org, "gaia", nil)
 	if err != nil {
 		err = fmt.Errorf("ListReleases failed: %w", err)
 		return
@@ -40,8 +40,8 @@ func GetPreviousMajorMinor(ctx context.Context, testVersion string) (previousVer
 		err = fmt.Errorf("failed to parse major version: %w", err)
 		return
 	}
-	semvers := make([]string, 0, len(releaes))
-	for _, release := range releaes {
+	semvers := make([]string, 0, len(releases))
+	for _, release := range releases {
 		semvers = append(semvers, release.GetTagName())
 	}
 	var previousMinor, previousRc bool
@@ -97,6 +97,7 @@ func GetSemverForBranch() (string, error) {
 
 func GetTestList() ([]string, error) {
 	retval := []string{}
+	uniq := map[string]bool{}
 	cmd := exec.Command("go", "test", "-list=.", "./...")
 	out, err := cmd.Output()
 	if err != nil {
@@ -105,8 +106,9 @@ func GetTestList() ([]string, error) {
 	}
 	lines := strings.Split(string(out), "\n")
 	for _, line := range lines {
-		if strings.HasPrefix(line, "Test") {
+		if strings.HasPrefix(line, "Test") && !uniq[line] {
 			retval = append(retval, line)
+			uniq[line] = true
 		}
 	}
 	rand.Shuffle(len(retval), func(i, j int) {