Add logging and exception handling to Data Hub request functions

app/enquiries/common/datahub_utils.py

@@ -1,18 +1,19 @@
 import logging
 import os
 import requests
-from requests_hawk import HawkAuth
+
 from cache_memoize import cache_memoize
 from datetime import date
 from django.conf import settings
 from django.forms.models import model_to_dict
-from requests.exceptions import RequestException
-from urllib.error import HTTPError
+from requests_hawk import HawkAuth
 
 import app.enquiries.ref_data as ref_data
 from app.enquiries.utils import get_oauth_payload
 from app.enquiries.common.cache import cached_requests
 
+logger = logging.getLogger(__name__)
+
 
 def dh_request(
     request,
@@ -68,18 +69,19 @@ def dh_request(
         }
 
     params = params if params else {}
-
     try:
         if method == "GET":
             response = requests.get(url, headers=headers, params=params, timeout=timeout)
         elif method == "POST":
             response = requests.post(url, headers=headers, json=payload, timeout=timeout)
-    except RequestException as e:
-        logging.error(f"Error {e} while requesting {url}, request timeout set to {timeout} secs")
+        return response
+    except Exception as e:
+        logger.error(
+            f"Error while requesting {url}: {e}"
+            f"; request timeout set to {timeout} secs"
+        )
         raise e
 
-    return response
-
 
 @cache_memoize(60 * 60)
 def fetch_metadata(name):
@@ -89,25 +91,32 @@ def fetch_metadata(name):
 
     :param name:
         The trailing part of the |data-hub-api|_ metadata endpoint e.g.
-        ``https://api.datahub.dev.uktrade.io/v4/metadata/<name>``.
+        ``https://api.dev.datahub.uktrade.digital/v4/metadata/<name>``.
     :type name: str
 
     :returns: The parsed metadata as a ``list`` of dictionaries.
     """
     url = os.path.join(settings.DATA_HUB_METADATA_URL, name)
-    response = cached_requests.get(
-        url,
-        auth=HawkAuth(
-            id=settings.DATA_HUB_ENQUIRY_MGMT_HAWK_ID,
-            key=settings.DATA_HUB_ENQUIRY_MGMT_HAWK_SECRET_KEY,
-        ),
-        # Add dummy data to avoid error: MissingContent payload content and/or content_type cannot
-        # be empty when always_hash_content is True
-        data={"data": name},
-        timeout=10,
-    )
-    response.raise_for_status()
-    return response.json()
+    try:
+        response = cached_requests.get(
+            url,
+            auth=HawkAuth(
+                id=settings.DATA_HUB_ENQUIRY_MGMT_HAWK_ID,
+                key=settings.DATA_HUB_ENQUIRY_MGMT_HAWK_SECRET_KEY,
+            ),
+            # Add dummy data to avoid error: MissingContent payload content and/or
+            # content_type cannot be empty when always_hash_content is True
+            data={"data": name},
+            timeout=10,
+        )
+        response.raise_for_status()
+        return response.json()
+    except Exception as e:
+        logger.error(
+            f"Error fetching metadata for {name} from {url}: {e}"
+            f"; response body: {response.json()}"
+        )
+        raise
 
 
 def resolve_metadata_id(title, metadata):
@@ -681,29 +690,31 @@ def dh_investment_create(request, enquiry):
     client_relationship_manager_id = dh_status["adviser"]
 
     url = settings.DATA_HUB_INVESTMENT_CREATE_URL
-    payload, error_key = dh_prepare_payload(
-        enquiry,
-        company_id,
-        contact_id,
-        referral_adviser,
-        client_relationship_manager_id,
-    )
-    if error_key:
-        response["errors"].append({error_key: "Reference data mismatch in DataHub"})
+    try:
+        payload, error_key = dh_prepare_payload(
+            enquiry,
+            company_id,
+            contact_id,
+            referral_adviser,
+            client_relationship_manager_id,
+        )
+        if error_key:
+            response["errors"].append({error_key: "Reference data mismatch in DataHub"})
+            return response
+    except Exception as e:
+        message = f"Error preparing payload for Data Hub: {e}"
+        logger.error(message)
+        response["errors"].append({"investment_create": message})
         return response
 
     try:
         result = dh_request(request, access_token, "POST", url, payload)
-
         result.raise_for_status()
-
         response["result"] = result.json()
-    except HTTPError as e:
-        response["errors"].append(
-            {"investment_create": f"Error contacting DataHub_ to create investment, {str(e)}"}
-        )
     except Exception as e:
-        response["errors"].append({"investment_create": f"Error creating investment, {str(e)}"})
+        message = f"Error contacting Data Hub to create investment: {e}"
+        logger.error(message)
+        response["errors"].append({"investment_create": message})
         return response
 
     if result.ok:

app/enquiries/tests/test_dh_integration.py

@@ -1,4 +1,6 @@
+import os
 import pytest
+import requests
 import requests_mock
 
 from datetime import date
@@ -12,6 +14,7 @@
 from app.enquiries.tests.factories import EnquiryFactory
 from app.enquiries.common.datahub_utils import (
     dh_request,
+    fetch_metadata,
     dh_company_search,
     dh_get_company_contact_list,
     dh_investment_create,
@@ -125,6 +128,45 @@ def test_dh_request_payload(self, fetch_metadata):
         assert payload == expected_dh_payload
         assert error_key is None
 
+    def test_fetch_metadata_raises_error(self):
+        metadata_name = "investment-specific-programme"
+        response_json = {'detail': 'Incorrect authentication credentials.'}
+        with requests_mock.Mocker() as m:
+            url = os.path.join(settings.DATA_HUB_METADATA_URL, metadata_name)
+            m.get(url, status_code=401, json=response_json)
+            with (
+                self.assertLogs("app.enquiries.common.datahub_utils", level="ERROR") as log,
+                pytest.raises(requests.exceptions.HTTPError)
+            ):
+                fetch_metadata(metadata_name)
+        assert any(
+            f"Error fetching metadata for {metadata_name} from {url}"
+            in message for message in log.output
+        )
+        assert any(
+            f"; response body: {response_json}"
+            in message for message in log.output
+        )
+
+    def test_dh_request_raises_error(self):
+        url = settings.DATA_HUB_COMPANY_SEARCH_URL
+        timeout = 2
+        with (
+            self.assertLogs("app.enquiries.common.datahub_utils", level="ERROR") as log,
+            pytest.raises(Exception),
+        ):
+            dh_request(
+                "mock_request", "access_token", "PATCH", url, {"key": "value"}, timeout=timeout,
+            )
+        assert any(
+            f"Error while requesting {url}"
+            in message for message in log.output
+        )
+        assert any(
+            f"; request timeout set to {timeout} secs"
+            in message for message in log.output
+        )
+
     @mock.patch("requests.post")
     def test_dh_request_timeout(self, mock_post):
         """ Tests to ensure data hub requests raise exception """
@@ -213,3 +255,27 @@ def test_investment_enquiry_cannot_submit_twice(self):
                 f"Enquiry can only be submitted once, previously submitted on {prev_date}, stage\
  {stage}",
             )
+
+    @mock.patch("app.enquiries.common.datahub_utils.dh_prepare_contact")
+    @mock.patch("app.enquiries.common.datahub_utils.dh_enquiry_readiness")
+    @mock.patch("app.enquiries.common.datahub_utils.dh_get_user_details")
+    @mock.patch("app.enquiries.common.datahub_utils.get_oauth_payload")
+    def test_investment_create_logs_error_when_failing_to_prepare_dh_payload(
+        self,
+        mock_oauth,
+        mock_get_user_details,
+        mock_enquiry_readiness,
+        mock_prepare_contact,
+    ):
+        mock_oauth.return_value = {"access_token": "mock_token"}
+        mock_get_user_details.return_value = ({"id": str(uuid4())}, None)
+        mock_enquiry_readiness.return_value = {"errors": [], "adviser": uuid4()}
+        mock_prepare_contact.return_value = (str(uuid4()), None)
+        with requests_mock.Mocker() as m:
+            url = os.path.join(settings.DATA_HUB_METADATA_URL, "investment-specific-programme")
+            m.get(url, status_code=401, json={"detail": "Incorrect authentication credentials."})
+            with self.assertLogs("app.enquiries.common.datahub_utils", level="ERROR") as log:
+                response = dh_investment_create("mock_request", EnquiryFactory())
+                assert "Error preparing payload for Data Hub" \
+                    in response["errors"][0]["investment_create"]
+        assert any("Error preparing payload for Data Hub" in message for message in log.output)