Cost-Centers rework: Scheduling (#532)

* Cost-Centers rework: Scheduling

* sub module commit changes

* resolving comments

* Removing old code (#533)

* Removing old code

* removing test

* adding test (#534)

* adding test

* increasing cov

* Adding migration scripts (#535)

* sub module branch change to master

* script changes for deleting old schedules

apps/mappings/queues.py

@@ -25,16 +25,6 @@ def async_auto_create_expense_field_mapping(mapping_setting: MappingSetting):
     async_task('apps.mappings.tasks.auto_create_expense_fields_mappings', int(mapping_setting.workspace_id), mapping_setting.destination_field, mapping_setting.source_field)
 
 
-def schedule_cost_centers_creation(import_to_fyle, workspace_id):
-    if import_to_fyle:
-        schedule, _ = Schedule.objects.update_or_create(func='apps.mappings.tasks.auto_create_cost_center_mappings', args='{}'.format(workspace_id), defaults={'schedule_type': Schedule.MINUTES, 'minutes': 24 * 60, 'next_run': datetime.now()})
-    else:
-        schedule: Schedule = Schedule.objects.filter(func='apps.mappings.tasks.auto_create_cost_center_mappings', args='{}'.format(workspace_id)).first()
-
-        if schedule:
-            schedule.delete()
-
-
 def schedule_fyle_attributes_creation(workspace_id: int):
     mapping_settings = MappingSetting.objects.filter(is_custom=True, import_to_fyle=True, workspace_id=workspace_id).all()
     if mapping_settings:
@@ -137,7 +127,7 @@ def construct_tasks_and_chain_import_fields_to_fyle(workspace_id):
     # For now we are only adding PROJECTS support that is why we are hardcoding it
     if mapping_settings:
         for mapping_setting in mapping_settings:
-            if mapping_setting.source_field in ['PROJECT']:
+            if mapping_setting.source_field in ['PROJECT', 'COST_CENTER']:
                 task_settings['mapping_settings'].append({
                     'source_field': mapping_setting.source_field,
                     'destination_field': mapping_setting.destination_field,

apps/mappings/schedules.py

@@ -31,7 +31,7 @@ def schedule_or_delete_fyle_import_tasks(workspace_general_settings: WorkspaceGe
     import_fields_count = MappingSetting.objects.filter(
         import_to_fyle=True,
         workspace_id=workspace_general_settings.workspace_id,
-        source_field__in=['PROJECT']
+        source_field__in=['PROJECT', 'COST_CENTER']
     ).count()
 
     # If the import fields count is 0, delete the schedule

apps/mappings/signals.py

@@ -7,7 +7,6 @@
 
 from apps.mappings.queues import (
     async_auto_create_expense_field_mapping,
-    schedule_cost_centers_creation,
     schedule_fyle_attributes_creation,
 )
 from apps.mappings.tasks import upload_attributes_to_fyle
@@ -51,12 +50,9 @@ def run_post_mapping_settings_triggers(sender, instance: MappingSetting, **kwarg
     """
     workspace_general_settings = WorkspaceGeneralSettings.objects.filter(workspace_id=instance.workspace_id).first()
 
-    if instance.source_field == 'PROJECT':
+    if instance.source_field in ['PROJECT', 'COST_CENTER']:
         new_schedule_or_delete_fyle_import_tasks(workspace_general_settings, instance)
 
-    if instance.source_field == 'COST_CENTER':
-        schedule_cost_centers_creation(instance.import_to_fyle, int(instance.workspace_id))
-
     if instance.is_custom:
         schedule_fyle_attributes_creation(int(instance.workspace_id))
 

apps/mappings/tasks.py

@@ -359,24 +359,6 @@ def sync_qbo_attribute(qbo_attribute_type: str, workspace_id: int):
         qbo_connection.sync_vendors()
 
 
-def create_fyle_cost_centers_payload(qbo_attributes: List[DestinationAttribute], existing_fyle_cost_centers: list):
-    """
-    Create Fyle Cost Centers Payload from QBO Objects
-    :param existing_fyle_cost_centers: Existing cost center names
-    :param qbo_attributes: QBO Objects
-    :return: Fyle Cost Centers Payload
-    """
-    fyle_cost_centers_payload = []
-
-    for qbo_attribute in qbo_attributes:
-        if qbo_attribute.value not in existing_fyle_cost_centers:
-            fyle_cost_centers_payload.append(
-                {'name': qbo_attribute.value, 'is_enabled': True if qbo_attribute.active is None else qbo_attribute.active, 'description': 'Cost Center - {0}, Id - {1}'.format(qbo_attribute.value, qbo_attribute.destination_id)}
-            )
-
-    return fyle_cost_centers_payload
-
-
 def create_fyle_tax_group_payload(qbo_attributes: List[DestinationAttribute], existing_fyle_tax_groups: list):
     """
     Create Fyle Cost Centers Payload from QBO Objects
@@ -393,46 +375,6 @@ def create_fyle_tax_group_payload(qbo_attributes: List[DestinationAttribute], ex
     return fyle_tax_group_payload
 
 
-def post_cost_centers_in_batches(platform: PlatformConnector, workspace_id: int, qbo_attribute_type: str):
-    existing_cost_center_names = ExpenseAttribute.objects.filter(attribute_type='COST_CENTER', workspace_id=workspace_id).values_list('value', flat=True)
-
-    qbo_attributes_count = DestinationAttribute.objects.filter(attribute_type=qbo_attribute_type, workspace_id=workspace_id).count()
-
-    page_size = 200
-
-    for offset in range(0, qbo_attributes_count, page_size):
-        limit = offset + page_size
-        paginated_qbo_attributes = DestinationAttribute.objects.filter(attribute_type=qbo_attribute_type, workspace_id=workspace_id).order_by('value', 'id')[offset:limit]
-
-        paginated_qbo_attributes = remove_duplicates(paginated_qbo_attributes)
-
-        fyle_payload: List[Dict] = create_fyle_cost_centers_payload(paginated_qbo_attributes, existing_cost_center_names)
-
-        if fyle_payload:
-            platform.cost_centers.post_bulk(fyle_payload)
-            platform.cost_centers.sync()
-
-        Mapping.bulk_create_mappings(paginated_qbo_attributes, 'COST_CENTER', qbo_attribute_type, workspace_id)
-
-
-@handle_import_exceptions(task_name='Auto Create Cost Center Mappings')
-def auto_create_cost_center_mappings(workspace_id):
-    """
-    Create Cost Center Mappings
-    """
-    fyle_credentials: FyleCredential = FyleCredential.objects.get(workspace_id=workspace_id)
-
-    platform = PlatformConnector(fyle_credentials)
-
-    mapping_setting = MappingSetting.objects.get(source_field='COST_CENTER', import_to_fyle=True, workspace_id=workspace_id)
-
-    platform.cost_centers.sync()
-
-    sync_qbo_attribute(mapping_setting.destination_field, workspace_id)
-
-    post_cost_centers_in_batches(platform, workspace_id, mapping_setting.destination_field)
-
-
 def create_fyle_expense_custom_field_payload(qbo_attributes: List[DestinationAttribute], workspace_id: int, fyle_attribute: str, platform: PlatformConnector, source_placeholder: str = None):
     """
     Create Fyle Expense Custom Field Payload from QBO Objects

apps/quickbooks_online/actions.py

@@ -14,7 +14,9 @@
 from apps.fyle.models import ExpenseGroup
 from apps.fyle.actions import update_complete_expenses
 from apps.quickbooks_online.utils import QBOConnector
+from apps.mappings.helpers import get_auto_sync_permission
 from apps.tasks.models import TaskLog
+from apps.workspaces.models import WorkspaceGeneralSettings
 from apps.workspaces.models import LastExportDetail, QBOCredential, Workspace
 
 from .helpers import generate_export_type_and_id
@@ -24,6 +26,17 @@
 logger.level = logging.INFO
 
 
+SYNC_METHODS = {
+    'ACCOUNT': 'accounts',
+    'ITEM': 'items',
+    'VENDOR': 'vendors',
+    'DEPARTMENT': 'departments',
+    'TAX_CODE': 'tax_codes',
+    'CLASS': 'classes',
+    'CUSTOMER': 'customers',
+}
+
+
 def update_last_export_details(workspace_id):
     last_export_detail = LastExportDetail.objects.get(workspace_id=workspace_id)
 
@@ -63,13 +76,26 @@ def refresh_quickbooks_dimensions(workspace_id: int):
     quickbooks_connector = QBOConnector(quickbooks_credentials, workspace_id=workspace_id)
 
     mapping_settings = MappingSetting.objects.filter(workspace_id=workspace_id, import_to_fyle=True)
+    credentials = QBOCredential.objects.get(workspace_id=workspace_id)
+    workspace_general_settings = WorkspaceGeneralSettings.objects.get(workspace_id=workspace_id)
+
     chain = Chain()
 
     for mapping_setting in mapping_settings:
-        if mapping_setting.source_field == 'PROJECT':
-            chain.append('apps.mappings.tasks.auto_import_and_map_fyle_fields', int(workspace_id))
-        elif mapping_setting.source_field == 'COST_CENTER':
-            chain.append('apps.mappings.tasks.auto_create_cost_center_mappings', int(workspace_id))
+        if mapping_setting.source_field in ['PROJECT', 'COST_CENTER']:
+            chain.append(
+                'fyle_integrations_imports.tasks.trigger_import_via_schedule',
+                workspace_id,
+                mapping_setting.destination_field,
+                mapping_setting.source_field,
+                'apps.quickbooks_online.utils.QBOConnector',
+                credentials,
+                [SYNC_METHODS[mapping_setting.destination_field]],
+                get_auto_sync_permission(workspace_general_settings, mapping_setting),
+                False,
+                None,
+                False
+            )
         elif mapping_setting.is_custom:
             chain.append('apps.mappings.tasks.async_auto_create_custom_field_mappings', int(workspace_id))
 

apps/workspaces/apis/import_settings/triggers.py

@@ -6,7 +6,6 @@
 from apps.fyle.models import ExpenseGroupSettings
 from apps.mappings.helpers import schedule_or_delete_fyle_import_tasks
 from apps.mappings.queues import (
-    schedule_cost_centers_creation,
     schedule_fyle_attributes_creation,
     schedule_tax_groups_creation,
 )
@@ -100,15 +99,6 @@ def pre_save_mapping_settings(self):
         """
         mapping_settings = self.__mapping_settings
 
-        cost_center_mapping_available = False
-
-        for setting in mapping_settings:
-            if setting['source_field'] == 'COST_CENTER':
-                cost_center_mapping_available = True
-
-        if not cost_center_mapping_available:
-            schedule_cost_centers_creation(False, self.__workspace_id)
-
         schedule_fyle_attributes_creation(self.__workspace_id)
 
         # Removal of department grouping will be taken care from post_delete() signal

scripts/python/create-update-new-categories-import.py

@@ -9,8 +9,6 @@
 try:
     # Create/update new schedules in a transaction block
     with transaction.atomic():
-        deleted_count = 0
-        updated_count = 0
         for schedule in existing_import_enabled_schedules:
             random_number = random.randint(1, 23)
             configuration = WorkspaceGeneralSettings.objects.get(workspace_id=schedule['args'])
@@ -29,5 +27,7 @@
                     minutes=24 * 60,
                     next_run=datetime.now() + timedelta(hours=random_number)
                 )
+        # remove this sanity check after running this script
+        raise Exception("This is a sanity check")
 except Exception as e:
     print(e)

scripts/python/create-update-new-cost-centers-import.py

@@ -0,0 +1,43 @@
+from django.db import transaction
+import random
+from datetime import datetime, timedelta
+from django_q.models import Schedule
+from fyle_accounting_mappings.models import MappingSetting
+
+existing_import_enabled_schedules = Schedule.objects.filter(
+    func__in=['apps.mappings.tasks.auto_create_cost_center_mappings']
+).values('args')
+
+try:
+    with transaction.atomic():
+        for schedule in existing_import_enabled_schedules:
+            random_number = random.randint(1, 23)
+            mapping_setting = MappingSetting.objects.filter(source_field='COST_CENTER', workspace_id=schedule['args'], import_to_fyle=True).first()
+            if mapping_setting:
+                print('Creating schedule for workspace_id: ', schedule['args'])
+                # adding the new schedule
+                Schedule.objects.update_or_create(
+                    func='apps.mappings.queues.construct_tasks_and_chain_import_fields_to_fyle',
+                    args=schedule['args'],
+                    defaults={
+                        'schedule_type': Schedule.MINUTES,
+                        'minutes':24 * 60,
+                        'next_run':datetime.now() + timedelta(hours=random_number)
+                    }
+                )
+                # deleting the old schedule
+                Schedule.objects.filter(
+                    func='apps.mappings.tasks.auto_create_cost_center_mappings',
+                    args=schedule['args']
+                ).delete()
+        # remove this sanity check after running this script
+        raise Exception("This is a sanity check")
+except Exception as e:
+    print(e)
+
+
+# Run this in sql
+# select * from django_q_schedule where func = 'apps.mappings.tasks.auto_create_cost_center_mappings';
+# --rows should be 0
+# If not check the workspace_id and delete the row
+# delete from django_q_schedule where func = 'apps.mappings.tasks.auto_create_cost_center_mappings' and args = 'workspace_id';