[viable/strict] Ignore failing jobs with unstable issues (#6362)

Allows the viable strict promotion script to use unstable issues. Jobs
that have unstable issues open will be ignored in viable strict
promotion.

Tested with 
0ef2e938d0a9a4b90434f98b5e128d0ffacaae26 (passed, only thing failing is
libtorch debug build which has an issue right now)
96afa8a2bb78e5410a83038fd1e3f83911601700 (failed since there's something
pending)
c5d92edd5acfa56bae4f0c1057d667c6356fd6c1 (failed since lint failed)

tools/scripts/fetch_latest_green_commit.py

@@ -1,6 +1,7 @@
 import json
 import re
 import sys
+from functools import lru_cache
 from pathlib import Path
 from typing import Any, cast, Dict, List, NamedTuple, Optional, Tuple
 
@@ -80,6 +81,23 @@ def get_commit_results(
     return workflow_checks
 
 
+@lru_cache
+def fetch_unstable_issues() -> List[str]:
+    issues = query_clickhouse_saved("issue_query", {"label": "unstable"})
+    return [
+        issue["title"][len("UNSTABLE") :].strip()
+        for issue in issues
+        if issue["title"].startswith("UNSTABLE") and issue["state"] == "open"
+    ]
+
+
+def is_unstable(job: dict[str, Any]) -> bool:
+    # Check if the job is an unstable job, either by name or by issue
+    if "unstable" in job["jobName"]:
+        return True
+    return job["name"] in fetch_unstable_issues()
+
+
 def is_green(
     commit: str, requires: List[str], results: List[Dict[str, Any]]
 ) -> Tuple[bool, str]:
@@ -88,9 +106,9 @@ def is_green(
     regex = {check: False for check in requires}
 
     for check in workflow_checks:
-        jobName = check["jobName"]
+        jobName = check["name"]
         # Ignore result from unstable job, be it success or failure
-        if "unstable" in jobName:
+        if "unstable" in jobName or jobName in fetch_unstable_issues():
             continue
 
         workflow_name = check["workflowName"]

tools/tests/test_fetch_latest_green_commit.py

@@ -47,12 +47,18 @@ def make_test_checks(self) -> List[Dict[str, Any]]:
         return workflow_checks
 
 
+@mock.patch(
+    "tools.scripts.fetch_latest_green_commit.fetch_unstable_issues",
+    return_value=[],
+)
 class TestPrintCommits(TestCase):
     @mock.patch(
         "tools.scripts.fetch_latest_green_commit.get_commit_results",
         return_value=TestChecks().make_test_checks(),
     )
-    def test_all_successful(self, mock_get_commit_results: Any) -> None:
+    def test_all_successful(
+        self, mock_get_commit_results: Any, mock_fetch_unstable_issues: Any
+    ) -> None:
         """Test with workflows are successful"""
         workflow_checks = mock_get_commit_results()
         self.assertTrue(is_green("sha", requires, workflow_checks)[0])
@@ -61,7 +67,9 @@ def test_all_successful(self, mock_get_commit_results: Any) -> None:
         "tools.scripts.fetch_latest_green_commit.get_commit_results",
         return_value=TestChecks().make_test_checks(),
     )
-    def test_necessary_successful(self, mock_get_commit_results: Any) -> None:
+    def test_necessary_successful(
+        self, mock_get_commit_results: Any, mock_fetch_unstable_issues: Any
+    ) -> None:
         """Test with necessary workflows are successful"""
         workflow_checks = mock_get_commit_results()
         workflow_checks = set_workflow_job_status(
@@ -85,7 +93,9 @@ def test_necessary_successful(self, mock_get_commit_results: Any) -> None:
         "tools.scripts.fetch_latest_green_commit.get_commit_results",
         return_value=TestChecks().make_test_checks(),
     )
-    def test_necessary_skipped(self, mock_get_commit_results: Any) -> None:
+    def test_necessary_skipped(
+        self, mock_get_commit_results: Any, mock_fetch_unstable_issues: Any
+    ) -> None:
         """Test with necessary job (ex: pull) skipped"""
         workflow_checks = mock_get_commit_results()
         workflow_checks = set_workflow_job_status(workflow_checks, "pull", "skipped")
@@ -96,7 +106,9 @@ def test_necessary_skipped(self, mock_get_commit_results: Any) -> None:
         "tools.scripts.fetch_latest_green_commit.get_commit_results",
         return_value=TestChecks().make_test_checks(),
     )
-    def test_skippable_skipped(self, mock_get_commit_results: Any) -> None:
+    def test_skippable_skipped(
+        self, mock_get_commit_results: Any, mock_fetch_unstable_issues: Any
+    ) -> None:
         """Test with skippable jobs (periodic and docker-release-builds skipped"""
         workflow_checks = mock_get_commit_results()
         workflow_checks = set_workflow_job_status(
@@ -111,7 +123,9 @@ def test_skippable_skipped(self, mock_get_commit_results: Any) -> None:
         "tools.scripts.fetch_latest_green_commit.get_commit_results",
         return_value=TestChecks().make_test_checks(),
     )
-    def test_necessary_failed(self, mock_get_commit_results: Any) -> None:
+    def test_necessary_failed(
+        self, mock_get_commit_results: Any, mock_fetch_unstable_issues: Any
+    ) -> None:
         """Test with necessary job (ex: Lint) failed"""
         workflow_checks = mock_get_commit_results()
         workflow_checks = set_workflow_job_status(workflow_checks, "Lint", "failed")
@@ -123,7 +137,9 @@ def test_necessary_failed(self, mock_get_commit_results: Any) -> None:
         "tools.scripts.fetch_latest_green_commit.get_commit_results",
         return_value=TestChecks().make_test_checks(),
     )
-    def test_skippable_failed(self, mock_get_commit_results: Any) -> None:
+    def test_skippable_failed(
+        self, mock_get_commit_results: Any, mock_fetch_unstable_issues: Any
+    ) -> None:
         """Test with failing skippable jobs (ex: docker-release-builds) should pass"""
         workflow_checks = mock_get_commit_results()
         workflow_checks = set_workflow_job_status(
@@ -138,7 +154,9 @@ def test_skippable_failed(self, mock_get_commit_results: Any) -> None:
     @mock.patch(
         "tools.scripts.fetch_latest_green_commit.get_commit_results", return_value={}
     )
-    def test_no_workflows(self, mock_get_commit_results: Any) -> None:
+    def test_no_workflows(
+        self, mock_get_commit_results: Any, mock_fetch_unstable_issues: Any
+    ) -> None:
         """Test with missing workflows"""
         workflow_checks = mock_get_commit_results()
         result = is_green("sha", requires, workflow_checks)