Explicitly Deselect Unselected Fields (#266)

* Bump version 3.1.2 * Explicitly deselect unselected fields * Fix dependabot issue --------- Co-authored-by: RushiT0122 <[email protected]>
singer-io · Feb 20, 2025 · 155e753 · 155e753
1 parent 67324fc
commit 155e753
Show file tree

Hide file tree

Showing 9 changed files with 252 additions and 8 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # Changelog
 
+## 3.2.0
+  * Add support for `select_fields_by_default` config property [#266](https://github.com/singer-io/tap-hubspot/pull/266)
+  * Fix dependabot issue
+
 ## 3.1.1
   * Replace legacy properties for Contacts and Deals [#265](https://github.com/singer-io/tap-hubspot/pull/265)
 
@@ -20,7 +24,7 @@
 
 ## 2.12.2
   * Use engagements_page_size advanced option [#234](https://github.com/singer-io/tap-hubspot/pull/234)
-  * 
+
 ## 2.12.1
   * Use sync start time for writing bookmarks [#226](https://github.com/singer-io/tap-hubspot/pull/226)
 

diff --git a/setup.py b/setup.py
@@ -3,7 +3,7 @@
 from setuptools import setup
 
 setup(name='tap-hubspot',
-      version='3.1.1',
+      version='3.2.0',
       description='Singer.io tap for extracting data from the HubSpot API',
       author='Stitch',
       url='http://singer.io',
@@ -12,7 +12,7 @@
       install_requires=[
           'attrs==16.3.0',
           'singer-python==5.13.0',
-          'requests==2.20.0',
+          'requests==2.32.3',
           'backoff==1.8.0',
           'requests_mock==1.3.0',
       ],

diff --git a/tap_hubspot/__init__.py b/tap_hubspot/__init__.py
@@ -64,6 +64,7 @@ class StateFields:
     "start_date": None,
     "hapikey": None,
     "include_inactives": None,
+    "select_fields_by_default": None,
 }
 
 ENDPOINTS = {
@@ -1316,7 +1317,25 @@ def get_selected_streams(remaining_streams, ctx):
             selected_streams.append(stream)
     return selected_streams
 
+def deselect_unselected_fields(catalog):
+    """
+    If a field isn't manually deselected, it will be included in the sync by default,
+    so we must explicitly deselect any such fields in the catalog.
+    """
+    LOGGER.info("Deselecting unselected fields")
+    for stream in catalog.get('streams'):
+        mdata = stream['metadata']
+        if mdata[0].get('metadata', {}).get('selected'):
+            for breadcrumb in mdata:
+                if breadcrumb.get('breadcrumb') and breadcrumb.get('metadata', {}).get('selected') is None:
+                    LOGGER.info("Deselecting %s", breadcrumb['breadcrumb'][1])
+                    breadcrumb['metadata']['selected'] = False
+
 def do_sync(STATE, catalog):
+    # If select_fields_by_default is not provided, default to True
+    if CONFIG.get('select_fields_by_default') is False:
+        deselect_unselected_fields(catalog)
+
     custom_objects = generate_custom_streams(mode="SYNC", catalog=catalog)
     # Clear out keys that are no longer used
     clean_state(STATE)
@@ -1467,6 +1486,13 @@ def main_impl():
     CONFIG.update(args.config)
     STATE = {}
 
+    if str(CONFIG.get('select_fields_by_default')).lower() not in ['none', 'true', 'false']:
+        raise ValueError(
+            "Invalid value for select_fields_by_default. It should be either 'true' or 'false'.")
+
+    CONFIG['select_fields_by_default'] = (
+        str(CONFIG.get('select_fields_by_default', 'true')).lower() != 'false')
+
     if args.state:
         STATE.update(args.state)
 

diff --git a/tests/test_hubspot_bookmarks.py b/tests/test_hubspot_bookmarks.py
@@ -38,7 +38,7 @@ def streams_to_test(self):
 
     def get_properties(self):
         return {
-            'start_date' : datetime.strftime(datetime.today()-timedelta(days=3), self.START_DATE_FORMAT),
+            'start_date' : datetime.strftime(datetime.today()-timedelta(days=5), self.START_DATE_FORMAT),
         }
 
     def setUp(self):

diff --git a/tests/test_hubspot_interrupted_sync.py b/tests/test_hubspot_interrupted_sync.py
@@ -50,7 +50,7 @@ def get_properties(self):
         # return {'start_date' : '2017-11-22T00:00:00Z'}
         return {
             'start_date' : datetime.strftime(
-                datetime.today()-timedelta(days=3), self.START_DATE_FORMAT
+                datetime.today()-timedelta(days=5), self.START_DATE_FORMAT
             ),
         }
 

diff --git a/tests/test_hubspot_interrupted_sync_offset.py b/tests/test_hubspot_interrupted_sync_offset.py
@@ -169,7 +169,7 @@ def name():
     def get_properties(self):
         return {
             'start_date' : datetime.strftime(
-                datetime.today()-timedelta(days=3), self.START_DATE_FORMAT
+                datetime.today()-timedelta(days=5), self.START_DATE_FORMAT
             ),
         }
 

diff --git a/tests/test_hubspot_pagination.py b/tests/test_hubspot_pagination.py
@@ -34,7 +34,7 @@ def streams_to_test(self):
 
     def get_properties(self):
         return {
-            'start_date' : datetime.strftime(datetime.today()-timedelta(days=7), self.START_DATE_FORMAT)
+            'start_date' : datetime.strftime(datetime.today()-timedelta(days=5), self.START_DATE_FORMAT)
         }
 
     def setUp(self):

diff --git a/tests/test_hubspot_start_date.py b/tests/test_hubspot_start_date.py
@@ -59,7 +59,7 @@ def get_properties(self, original=True):
             }
         else:
             return {
-                'start_date': self.timedelta_formatted(utc_today, days=-5)
+                'start_date': self.timedelta_formatted(utc_today, days=-3)
             }
 
     def test_run(self):

diff --git a/tests/unittests/test_deselect_unselected_fields.py b/tests/unittests/test_deselect_unselected_fields.py
@@ -0,0 +1,214 @@
+import unittest
+from unittest.mock import patch, MagicMock
+from tap_hubspot import deselect_unselected_fields, do_sync, main_impl, CONFIG
+
+
+class TestMainImpl(unittest.TestCase):
+
+    @patch('tap_hubspot.utils.parse_args')
+    @patch('tap_hubspot.do_discover')
+    @patch('tap_hubspot.do_sync')
+    def test_main_impl_default_behavior(self, mock_do_sync, mock_do_discover, mock_parse_args):
+        """Test the default behavior of the main_impl function when select_fields_by_default is not set."""
+        mock_args = MagicMock()
+        mock_args.config = {}
+        mock_args.state = None
+        mock_args.discover = False
+        mock_args.properties = None
+        mock_parse_args.return_value = mock_args
+
+        main_impl()
+
+        self.assertTrue(CONFIG['select_fields_by_default'])
+        mock_do_discover.assert_not_called()
+        mock_do_sync.assert_not_called()
+
+    @patch('tap_hubspot.utils.parse_args')
+    @patch('tap_hubspot.do_discover')
+    @patch('tap_hubspot.do_sync')
+    def test_main_impl_select_fields_by_default_true(self, mock_do_sync, mock_do_discover, mock_parse_args):
+        """Test the behavior of the main_impl function when select_fields_by_default is set to true."""
+        mock_args = MagicMock()
+        mock_args.config = {'select_fields_by_default': 'true'}
+        mock_args.state = None
+        mock_args.discover = False
+        mock_args.properties = None
+        mock_parse_args.return_value = mock_args
+
+        main_impl()
+
+        self.assertTrue(CONFIG['select_fields_by_default'])
+        mock_do_discover.assert_not_called()
+        mock_do_sync.assert_not_called()
+
+    @patch('tap_hubspot.utils.parse_args')
+    @patch('tap_hubspot.do_discover')
+    @patch('tap_hubspot.do_sync')
+    def test_main_impl_select_fields_by_default_false(self, mock_do_sync, mock_do_discover, mock_parse_args):
+        """Test the behavior of the main_impl function when select_fields_by_default is set to false."""
+        mock_args = MagicMock()
+        mock_args.config = {'select_fields_by_default': 'false'}
+        mock_args.state = None
+        mock_args.discover = False
+        mock_args.properties = None
+        mock_parse_args.return_value = mock_args
+
+        main_impl()
+
+        self.assertFalse(CONFIG['select_fields_by_default'])
+        mock_do_discover.assert_not_called()
+        mock_do_sync.assert_not_called()
+
+    @patch('tap_hubspot.utils.parse_args')
+    @patch('tap_hubspot.do_discover')
+    @patch('tap_hubspot.do_sync')
+    def test_main_impl_invalid_select_fields_by_default(self, mock_do_sync, mock_do_discover, mock_parse_args):
+        """Test the behavior of the main_impl function when select_fields_by_default is set to an invalid value."""
+        mock_args = MagicMock()
+        mock_args.config = {'select_fields_by_default': 'invalid'}
+        mock_args.state = None
+        mock_args.discover = False
+        mock_args.properties = None
+        mock_parse_args.return_value = mock_args
+
+        with self.assertRaises(ValueError):
+            main_impl()
+
+        mock_do_discover.assert_not_called()
+        mock_do_sync.assert_not_called()
+
+class TestDoSync(unittest.TestCase):
+
+    @patch('tap_hubspot.deselect_unselected_fields')
+    @patch('tap_hubspot.generate_custom_streams')
+    @patch('tap_hubspot.clean_state')
+    @patch('tap_hubspot.Context')
+    @patch('tap_hubspot.validate_dependencies')
+    @patch('tap_hubspot.get_streams_to_sync')
+    @patch('tap_hubspot.get_selected_streams')
+    @patch('tap_hubspot.singer')
+    def test_do_sync_select_fields_by_default_none(self, mock_singer, mock_get_selected_streams, mock_get_streams_to_sync, mock_validate_dependencies, mock_Context, mock_clean_state, mock_generate_custom_streams, mock_deselect_unselected_fields):
+        """Test the default behavior of the do_sync function. When select_fields_by_default is not specified, it should not call deselect_unselected_fields."""
+        # Mocking the necessary functions and objects
+        mock_singer.get_currently_syncing.return_value = None
+        mock_get_streams_to_sync.return_value = []
+        mock_get_selected_streams.return_value = []
+        mock_generate_custom_streams.return_value = []
+
+        # Mocking the catalog and state
+        CONFIG.update({'select_fields_by_default': None})
+        catalog = {'streams': []}
+        state = {}
+
+        # Call the function
+        do_sync(state, catalog)
+
+        # Assertions
+        mock_deselect_unselected_fields.assert_not_called()
+
+    # @patch('tap_hubspot.CONFIG', {'select_fields_by_default': True})
+    @patch('tap_hubspot.deselect_unselected_fields')
+    @patch('tap_hubspot.generate_custom_streams')
+    @patch('tap_hubspot.clean_state')
+    @patch('tap_hubspot.Context')
+    @patch('tap_hubspot.validate_dependencies')
+    @patch('tap_hubspot.get_streams_to_sync')
+    @patch('tap_hubspot.get_selected_streams')
+    @patch('tap_hubspot.singer')
+    def test_do_sync_select_fields_by_default_true(self, mock_singer, mock_get_selected_streams, mock_get_streams_to_sync, mock_validate_dependencies, mock_Context, mock_clean_state, mock_generate_custom_streams, mock_deselect_unselected_fields):
+        """Test the default behavior of the do_sync function. When select_fields_by_default is True, it should not call deselect_unselected_fields."""
+        # Mocking the necessary functions and objects
+        mock_singer.get_currently_syncing.return_value = None
+        mock_get_streams_to_sync.return_value = []
+        mock_get_selected_streams.return_value = []
+        mock_generate_custom_streams.return_value = []
+
+        # Mocking the catalog and state
+        CONFIG.update({'select_fields_by_default': 'true'})
+        catalog = {'streams': []}
+        state = {}
+
+        # Call the function
+        do_sync(state, catalog)
+
+        # Assertions
+        mock_deselect_unselected_fields.assert_not_called()
+
+    # @patch('tap_hubspot.CONFIG', {'select_fields_by_default': False})
+    @patch('tap_hubspot.deselect_unselected_fields')
+    @patch('tap_hubspot.generate_custom_streams')
+    @patch('tap_hubspot.clean_state')
+    @patch('tap_hubspot.Context')
+    @patch('tap_hubspot.validate_dependencies')
+    @patch('tap_hubspot.get_streams_to_sync')
+    @patch('tap_hubspot.get_selected_streams')
+    @patch('tap_hubspot.singer')
+    def test_do_sync_select_fields_by_default_false(self, mock_singer, mock_get_selected_streams, mock_get_streams_to_sync, mock_validate_dependencies, mock_Context, mock_clean_state, mock_generate_custom_streams, mock_deselect_unselected_fields):
+        """Test the default behavior of the do_sync function. When select_fields_by_default is False, it should call deselect_unselected_fields."""
+        # Mocking the necessary functions and objects
+        mock_singer.get_currently_syncing.return_value = None
+        mock_get_streams_to_sync.return_value = []
+        mock_get_selected_streams.return_value = []
+        mock_generate_custom_streams.return_value = []
+
+        # Mocking the catalog and state
+        CONFIG.update({'select_fields_by_default': 'false'})
+        catalog = {'streams': []}
+        state = {}
+
+        # Call the function
+        do_sync(state, catalog)
+
+        # Assertions
+        mock_deselect_unselected_fields.assert_called_once_with(catalog)
+
+
+class TestDeselectUnselectedFields(unittest.TestCase):
+
+    def test_deselect_unselected_fields(self):
+        catalog = {
+            'streams': [
+                {
+                    "stream_id": "test_stream_1",
+                    'metadata': [
+                        {'breadcrumb': [], 'metadata': {'selected': True}},
+                        {'breadcrumb': ['properties', 'field1'], 'metadata': {}},
+                        {'breadcrumb': ['properties', 'field2'], 'metadata': {'selected': True}},
+                        {'breadcrumb': ['properties', 'field3'], 'metadata': {'selected': False}}
+                    ]
+                },
+                {
+                    "stream_id": "test_stream_2",
+                    'metadata': [
+                        {'breadcrumb': [], 'metadata': {'selected': False}},
+                        {'breadcrumb': ['properties', 'field1'], 'metadata': {}},
+                        {'breadcrumb': ['properties', 'field2'], 'metadata': {}}
+                    ]
+                }
+            ]
+        }
+
+        expected_catalog = {
+            'streams': [
+                {
+                    "stream_id": "test_stream_1",
+                    'metadata': [
+                        {'breadcrumb': [], 'metadata': {'selected': True}},
+                        {'breadcrumb': ['properties', 'field1'], 'metadata': {'selected': False}},
+                        {'breadcrumb': ['properties', 'field2'], 'metadata': {'selected': True}},
+                        {'breadcrumb': ['properties', 'field3'], 'metadata': {'selected': False}}
+                    ]
+                },
+                {
+                    "stream_id": "test_stream_2",
+                    'metadata': [
+                        {'breadcrumb': [], 'metadata': {'selected': False}},
+                        {'breadcrumb': ['properties', 'field1'], 'metadata': {}},
+                        {'breadcrumb': ['properties', 'field2'], 'metadata': {}}
+                    ]
+                }
+            ]
+        }
+
+        deselect_unselected_fields(catalog)
+        self.assertEqual(catalog, expected_catalog)