Fix intx tests

.circleci/config.yml

@@ -46,7 +46,7 @@ workflows:
   commit:
     jobs:
       - build:
-          context: 
+          context:
           - circleci-user
           - tap-tester-user
   build_daily:
@@ -59,6 +59,6 @@ workflows:
                 - master
     jobs:
       - build:
-          context: 
+          context:
           - circleci-user
           - tap-tester-user

README.md

@@ -31,6 +31,7 @@ Config properties:
 | `start_date` | Y | "2010-01-01T00:00:00Z" | The default start date to use for date modified replication, when available. |
 | `user_agent` | N | "Vandelay Industries ETL Runner" | The user agent to send on every request. |
 | `request_timeout` | N | 300 | Time for which request should wait to get response. |
+| `chunk_size` | N | 100 | The length of a batch for 'reports_email_activity'. |
 
 ## Usage 
 

tap_mailchimp/__init__.py

@@ -21,8 +21,9 @@ def do_discover(client):
     LOGGER.info('Testing authentication')
     try:
         client.get('/lists', params={'count': 1})
-    except:
-        raise Exception('Error testing Mailchimp authentication')
+    except Exception as e:
+        msg  = 'Error testing Mailchimp authentication. Error: {}: {}'.format(e.__class__.__name__, str(e))
+        raise Exception(msg) from None
 
     LOGGER.info('Starting discover')
     catalog = discover()

tap_mailchimp/client.py

@@ -1,3 +1,4 @@
+import functools
 import backoff
 import requests
 import singer
@@ -7,12 +8,129 @@
 LOGGER = singer.get_logger()
 
 REQUEST_TIMEOUT = 300
+
 class ClientRateLimitError(Exception):
     pass
 
 class Server5xxError(Exception):
     pass
 
+class MailchimpError(Exception):
+    pass
+
+class MailchimpBadRequestError(MailchimpError):
+    pass
+
+class MailchimpUnAuthorizedError(MailchimpError):
+    pass
+
+class MailchimpForbiddenError(MailchimpError):
+    pass
+
+class MailchimpNotFoundError(MailchimpError):
+    pass
+
+class MailchimpMethodNotAllowedError(MailchimpError):
+    pass
+
+class MailchimpNestingError(MailchimpError):
+    pass
+
+class MailchimpInvalidMethodError(MailchimpError):
+    pass
+
+class MailchimpUpgradeError(MailchimpError):
+    pass
+
+class MailchimpRateLimitError(ClientRateLimitError):
+    pass
+
+class MailchimpInternalServerError(Server5xxError):
+    pass
+
+# Error glossary: https://mailchimp.com/developer/marketing/docs/errors/
+ERROR_CODE_EXCEPTION_MAPPING = {
+    400: {
+        "raise_exception": MailchimpBadRequestError,
+        "message": "Mailchimp Client faced a bad request."
+    },
+    401: {
+        "raise_exception": MailchimpUnAuthorizedError,
+        "message": "The API key is either invalid or disabled."
+    },
+    403: {
+        "raise_exception": MailchimpForbiddenError,
+        "message": "User does not have access to the requested operation."
+    },
+    404: {
+        "raise_exception": MailchimpNotFoundError,
+        "message": "The requested resource could not be found."
+    },
+    405: {
+        "raise_exception": MailchimpMethodNotAllowedError,
+        "message": "The resource does not accept the HTTP method."
+    },
+    414: {
+        "raise_exception": MailchimpNestingError,
+        "message": "The sub-resource requested is nested too deeply."
+    },
+    422: {
+        "raise_exception": MailchimpInvalidMethodError,
+        "message": "You can only use the X-HTTP-Method-Override header with the POST method."
+    },
+    426: {
+        "raise_exception": MailchimpUpgradeError,
+        "message": "Your request was made with the HTTP protocol. Please make your request via HTTPS rather than HTTP."
+    },
+    429: {
+        "raise_exception": MailchimpRateLimitError,
+        "message": "You have exceeded the limit of 10 simultaneous connections."
+    },
+    500: {
+        "raise_exception": MailchimpInternalServerError,
+        "message": "A deep internal error has occurred during the processing of your request."
+    }}
+
+def get_exception_for_error_code(status_code):
+    """Function to retrieve exceptions based on status_code"""
+
+    exception = ERROR_CODE_EXCEPTION_MAPPING.get(status_code, {}).get('raise_exception')
+    if not exception:
+        exception = Server5xxError if status_code > 500 else MailchimpError
+    return exception
+
+def raise_for_error(response):
+    """Function to raise an error by extracting the message from the error response"""
+    try:
+        json_response = response.json()
+
+    except Exception:
+        json_response = {}
+
+    status_code = response.status_code
+    msg = json_response.get(
+        "detail",
+        ERROR_CODE_EXCEPTION_MAPPING.get(status_code, {}).get(
+            "message", "Unknown Error"
+        )
+    )
+
+    message = "HTTP-error-code: {}, Error: {}".format(status_code, msg)
+
+    exc = get_exception_for_error_code(status_code)
+    raise exc(message) from None
+
+def retry_pattern(fnc):
+    """Function for backoff"""
+    @backoff.on_exception(backoff.expo,
+                          (Server5xxError, MailchimpRateLimitError, ConnectionError, Timeout),
+                          max_tries=6,
+                          factor=3)
+    @functools.wraps(fnc)
+    def wrapper(*args, **kwargs):
+        return fnc(*args, **kwargs)
+    return wrapper
+
 class MailchimpClient:
     def __init__(self, config):
         self.__user_agent = config.get('user_agent')
@@ -45,14 +163,6 @@ def get_base_url(self):
                             endpoint='base_url')
         self.__base_url = data['api_endpoint']
 
-    @backoff.on_exception(backoff.expo,
-                          Timeout, # Backoff for request timeout
-                          max_tries=5,
-                          factor=2)
-    @backoff.on_exception(backoff.expo,
-                          (Server5xxError, ClientRateLimitError, ConnectionError),
-                          max_tries=6,
-                          factor=3)
     def request(self, method, path=None, url=None, s3=False, **kwargs):
         if url is None and self.__base_url is None:
             self.get_base_url()
@@ -87,21 +197,18 @@ def request(self, method, path=None, url=None, s3=False, **kwargs):
             response = self.__session.request(method, url, timeout=self.__request_timeout, **kwargs) # Pass request timeout
             timer.tags[metrics.Tag.http_status_code] = response.status_code
 
-        if response.status_code >= 500:
-            raise Server5xxError()
-
-        if response.status_code == 429:
-            raise ClientRateLimitError()
-
-        response.raise_for_status()
+        if response.status_code != 200:
+            raise_for_error(response)
 
         if s3:
             return response
 
         return response.json()
 
+    @retry_pattern
     def get(self, path, **kwargs):
         return self.request('GET', path=path, **kwargs)
 
+    @retry_pattern
     def post(self, path, **kwargs):
         return self.request('POST', path=path, **kwargs)

tap_mailchimp/schema.py

@@ -40,6 +40,11 @@ def get_schemas():
             for replication_key in stream_obj.replication_keys:
                 mdata_map[('properties', replication_key)]['inclusion'] = 'automatic'
 
+        # Update inclusion for extra fields which we need to replicate data
+        if stream_obj.extra_automatic_fields:
+            for field in stream_obj.extra_automatic_fields:
+                mdata_map[('properties', field)]['inclusion'] = 'automatic'
+
         metadata_list = metadata.to_list(mdata_map)
         field_metadata[stream_name] = metadata_list
 

tap_mailchimp/schemas/reports_email_activity.json

@@ -2,6 +2,12 @@
   "type": "object",
   "additionalProperties": false,
   "properties": {
+    "_sdc_record_hash": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
     "campaign_id": {
       "type": [
         "string"