Merge pull request #162 from okurz/feature/dashboard_requests

Extract dashboard helper methods

openqabot/approver.py

@@ -13,6 +13,7 @@
 from openqa_client.exceptions import RequestError
 from openqabot.errors import NoResultsError
 from openqabot.openqa import openQAInterface
+from openqabot.dashboard import get_json
 
 from . import OBS_GROUP, OBS_MAINT_PRJ, OBS_URL, QEM_DASHBOARD
 from .loader.qem import (
@@ -23,7 +24,6 @@
     get_incidents_approver,
     get_single_incident,
 )
-from .utils import retry3 as requests
 
 log = getLogger("bot.approver")
 
@@ -133,9 +133,7 @@ def is_job_marked_acceptable_for_incident(self, job_id: int, inc: int) -> bool:
 
     @lru_cache(maxsize=128)
     def get_jobs(self, job_aggr: JobAggr, api: str, inc: int) -> bool:
-        results = requests.get(
-            QEM_DASHBOARD + api + str(job_aggr.id), headers=self.token
-        ).json()
+        results = get_json(api + str(job_aggr.id), headers=self.token)
 
         # keep jobs explicitly marked as acceptable for this incident by openQA comments
         for res in results:

openqabot/dashboard.py

@@ -0,0 +1,21 @@
+# Copyright SUSE LLC
+# SPDX-License-Identifier: MIT
+from typing import Dict
+import requests as req
+
+from . import QEM_DASHBOARD
+from .utils import retry5 as requests
+
+
+def get_json(route: str, **kwargs: Dict) -> str:
+    return requests.get(QEM_DASHBOARD + route, **kwargs).json()
+
+
+def patch(route: str, **kwargs: Dict) -> str:
+    # openqabot/loader/qem.py originally used req.patch so we will just use that here without retry. Can potentially be changed to used retry5 as requests as
+    # well
+    return req.patch(QEM_DASHBOARD + route, **kwargs)
+
+
+def put(route: str, **kwargs: Dict) -> str:
+    return requests.put(QEM_DASHBOARD + route, **kwargs)

openqabot/loader/qem.py

@@ -4,10 +4,7 @@
 from operator import itemgetter
 from pprint import pformat
 from typing import Dict, List, NamedTuple, Sequence
-
-import requests as req
-
-from .. import QEM_DASHBOARD
+from openqabot.dashboard import get_json, put, patch
 from ..errors import (
     EmptyChannels,
     EmptyPackagesError,
@@ -17,7 +14,6 @@
 )
 from ..types import Data
 from ..types.incident import Incident
-from ..utils import retry5 as requests
 
 log = getLogger("bot.loader.qem")
 
@@ -34,9 +30,7 @@ class JobAggr(NamedTuple):
 
 
 def get_incidents(token: Dict[str, str]) -> List[Incident]:
-    incidents = requests.get(
-        QEM_DASHBOARD + "api/incidents", headers=token, verify=True
-    ).json()
+    incidents = get_json("api/incidents", headers=token, verify=True)
 
     if "error" in incidents:
         raise Exception(incidents)
@@ -63,31 +57,27 @@ def get_incidents(token: Dict[str, str]) -> List[Incident]:
 
 def get_active_incidents(token: Dict[str, str]) -> Sequence[int]:
     try:
-        data = requests.get(QEM_DASHBOARD + "api/incidents", headers=token).json()
+        data = get_json("api/incidents", headers=token)
     except Exception as e:
         log.exception(e)
         raise e
     return list(set([i["number"] for i in data]))
 
 
 def get_incidents_approver(token: Dict[str, str]) -> List[IncReq]:
-    incidents = requests.get(QEM_DASHBOARD + "api/incidents", headers=token).json()
+    incidents = get_json("api/incidents", headers=token)
     return [IncReq(i["number"], i["rr_number"]) for i in incidents if i["inReviewQAM"]]
 
 
 def get_single_incident(token: Dict[str, str], incident_id: int) -> List[IncReq]:
-    incident = requests.get(
-        QEM_DASHBOARD + "api/incidents/" + incident_id, headers=token
-    ).json()
+    incident = get_json("api/incidents/" + incident_id, headers=token)
     return [IncReq(incident["number"], incident["rr_number"])]
 
 
 def get_incident_settings(
     inc: int, token: Dict[str, str], all_incidents: bool = False
 ) -> List[JobAggr]:
-    settings = requests.get(
-        QEM_DASHBOARD + "api/incident_settings/" + str(inc), headers=token
-    ).json()
+    settings = get_json("api/incident_settings/" + str(inc), headers=token)
     if not settings:
         raise NoResultsError(
             f"Inc {inc} does not have any job_settings. Consider adding package specific settings to the metadata repository."
@@ -104,10 +94,9 @@ def get_incident_settings(
 
 
 def get_incident_settings_data(token: Dict[str, str], number: int) -> Sequence[Data]:
-    url = QEM_DASHBOARD + "api/incident_settings/" + f"{number}"
     log.info("Getting settings for %s", number)
     try:
-        data = requests.get(url, headers=token).json()
+        data = get_json("api/incident_settings/" + f"{number}", headers=token)
     except Exception as e:
         log.exception(e)
         raise e
@@ -140,9 +129,7 @@ def get_incident_results(inc: int, token: Dict[str, str]):
     ret = []
     for job_aggr in settings:
         try:
-            data = requests.get(
-                QEM_DASHBOARD + "api/jobs/incident/" + f"{job_aggr.id}", headers=token
-            ).json()
+            data = get_json("api/jobs/incident/" + f"{job_aggr.id}", headers=token)
             ret += data
         except Exception as e:
             log.exception(e)
@@ -154,8 +141,7 @@ def get_incident_results(inc: int, token: Dict[str, str]):
 
 
 def get_aggregate_settings(inc: int, token: Dict[str, str]) -> List[JobAggr]:
-    url = QEM_DASHBOARD + "api/update_settings/" + str(inc)
-    settings = requests.get(url, headers=token).json()
+    settings = get_json("api/update_settings/" + str(inc), headers=token)
     if not settings:
         raise NoResultsError(f"Inc {inc} does not have any aggregates settings")
 
@@ -168,13 +154,9 @@ def get_aggregate_settings(inc: int, token: Dict[str, str]) -> List[JobAggr]:
 
 
 def get_aggregate_settings_data(token: Dict[str, str], data: Data):
-    url = (
-        QEM_DASHBOARD
-        + "api/update_settings"
-        + f"?product={data.product}&arch={data.arch}"
-    )
+    url = "api/update_settings" + f"?product={data.product}&arch={data.arch}"
     try:
-        settings = requests.get(url, headers=token).json()
+        settings = get_json(url, headers=token)
     except Exception as e:
         log.exception(e)
         raise e
@@ -214,9 +196,7 @@ def get_aggregate_results(inc: int, token: Dict[str, str]):
     ret = []
     for job_aggr in settings:
         try:
-            data = requests.get(
-                QEM_DASHBOARD + "api/jobs/update/" + f"{job_aggr.id}", headers=token
-            ).json()
+            data = get_json("api/jobs/update/" + f"{job_aggr.id}", headers=token)
         except Exception as e:
             log.exception(e)
             raise e
@@ -233,7 +213,7 @@ def update_incidents(token: Dict[str, str], data, **kwargs) -> int:
     while retry >= 0:
         retry -= 1
         try:
-            ret = req.patch(QEM_DASHBOARD + "api/incidents", headers=token, json=data)
+            ret = patch("api/incidents", headers=token, json=data)
         except Exception as e:  # pylint: disable=broad-except
             log.exception(e)
             return 1
@@ -251,7 +231,7 @@ def update_incidents(token: Dict[str, str], data, **kwargs) -> int:
 
 def post_job(token: Dict[str, str], data) -> None:
     try:
-        result = requests.put(QEM_DASHBOARD + "api/jobs", headers=token, json=data)
+        result = put("api/jobs", headers=token, json=data)
         if result.status_code != 200:
             log.error(result.text)
 
@@ -261,9 +241,7 @@ def post_job(token: Dict[str, str], data) -> None:
 
 def update_job(token: Dict[str, str], job_id: int, data) -> None:
     try:
-        result = requests.patch(
-            QEM_DASHBOARD + "api/jobs/" + str(job_id), headers=token, json=data
-        )
+        result = patch("api/jobs/" + str(job_id), headers=token, json=data)
         if result.status_code != 200:
             log.error(result.text)
 

openqabot/openqabot.py

@@ -4,12 +4,11 @@
 from logging import getLogger
 from os import environ
 
-from . import QEM_DASHBOARD
+from openqabot.dashboard import put
 from .errors import PostOpenQAError
 from .loader.config import get_onearch, load_metadata
 from .loader.qem import get_incidents
 from .openqa import openQAInterface
-from .utils import retry3 as requests
 
 log = getLogger("bot.openqabot")
 
@@ -40,9 +39,8 @@ def post_qem(self, data, api) -> None:
             )
             return
 
-        url = QEM_DASHBOARD + api
         try:
-            res = requests.put(url, headers=self.token, json=data)
+            res = put(api, headers=self.token, json=data)
             log.info(
                 "Put to dashboard result %s, database id: %s",
                 res.status_code,

openqabot/types/aggregate.py

@@ -11,7 +11,7 @@
 from ..errors import NoTestIssues, SameBuildExists
 from ..loader.repohash import merge_repohash
 from ..pc_helper import apply_pc_tools_image, apply_publiccloud_pint_image
-from ..utils import retry3 as requests
+from ..dashboard import get_json
 from .baseconf import BaseConf
 from .incident import Incident
 
@@ -124,11 +124,11 @@ def __call__(
             )
 
             try:
-                old_jobs = requests.get(
-                    QEM_DASHBOARD + "api/update_settings",
+                old_jobs = get_json(
+                    "api/update_settings",
                     params={"product": self.product, "arch": arch},
                     headers=token,
-                ).json()
+                )
             except Exception as e:  # pylint: disable=broad-except
                 log.exception(e)
                 old_jobs = None

tests/test_aggregate.py

@@ -59,7 +59,7 @@ def mock_get(*args, **kwargs):
         return MockResponse()
 
     monkeypatch.setattr(
-        "openqabot.types.aggregate.requests.get",
+        "openqabot.dashboard.requests.get",
         mock_get,
     )