Merge pull request #4 from fivetran/more_updates

More updates

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: servicenow_integration_tests_4
+      schema: servicenow_integration_tests_445
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: servicenow_integration_tests_4
+      schema: servicenow_integration_tests_445
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: servicenow_integration_tests_4
+      schema: servicenow_integration_tests_445
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: servicenow_integration_tests_4
+      schema: servicenow_integration_tests_445
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: servicenow_integration_tests_4
+      schema: servicenow_integration_tests_445
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -10,7 +10,7 @@ dispatch:
 
 vars:
   servicenow:
-    servicenow_schema: servicenow_integration_tests_3
+    servicenow_schema: servicenow_integration_tests_445
     servicenow_task_identifier: "task_data"
     servicenow_task_sla_identifier: "task_sla_data"
     servicenow_sys_user_identifier: "sys_user_data"
@@ -20,6 +20,9 @@ vars:
     servicenow_change_task_identifier: "change_task_data"
     servicenow_change_request_identifier: "change_request_data"
     servicenow_core_company_identifier: "core_company_data"
+    servicenow_cmdb_ci_identifier: "cmdb_ci_data"
+    servicenow_cmdb_ci_service_identifier: "cmdb_ci_service_data"
+    servicenow_sys_user_group_identifier: "sys_user_group_data"
 
 seeds:
   +quote_columns: "{{ true if target.type in ('redshift','postgres') else false }}"
@@ -29,6 +32,7 @@ seeds:
       sys_created_on: "timestamp"
       resolved_at: "timestamp"
       closed_at: "timestamp"
+      market_cap: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}"
       confirmed_by_value: "{{ 'varchar' if target.type in ['redshift', 'postgres', 'snowflake'] else 'string' }}"
       first_reported_by_task_value: "{{ 'varchar' if target.type in ['redshift', 'postgres', 'snowflake'] else 'string' }}"
       requested_by_value: "{{ 'varchar' if target.type in ['redshift', 'postgres', 'snowflake'] else 'string' }}"
@@ -44,7 +48,6 @@ seeds:
       opened_by_value: "{{ 'varchar' if target.type in ['redshift', 'postgres', 'snowflake'] else 'string' }}"
       sys_created_by: "{{ 'varchar' if target.type in ['redshift', 'postgres', 'snowflake'] else 'string' }}"
       sys_updated_by: "{{ 'varchar' if target.type in ['redshift', 'postgres', 'snowflake'] else 'string' }}"
-
     task_data:
       +enabled: "{{ true if target.type != 'snowflake' else false }}"
     task_data_snowflake:

integration_tests/seeds/cmdb_ci_data.csv

@@ -0,0 +1,2 @@
+sys_id,_fivetran_deleted,_fivetran_synced,asset_link,asset_tag,asset_value,assigned,assigned_to_link,assigned_to_value,assignment_group_link,assignment_group_value,attestation_score,attestation_status,attested,attested_by_link,attested_by_value,attested_date,attributes,business_unit_link,business_unit_value,can_print,category,change_control_link,change_control_value,checked_in,checked_out,comments,company_link,company_value,correlation_id,cost,cost_cc,cost_center_link,cost_center_value,delivery_date,department_link,department_value,discovery_source,dns_domain,due,due_in,duplicate_of_link,duplicate_of_value,environment,fault_count,first_discovered,fqdn,gl_account,install_date,install_status,invoice_number,ip_address,justification,last_discovered,lease_id,life_cycle_stage_link,life_cycle_stage_status_link,life_cycle_stage_status_value,life_cycle_stage_value,location_link,location_value,mac_address,maintenance_schedule_link,maintenance_schedule_value,managed_by_group_link,managed_by_group_value,managed_by_link,managed_by_value,manufacturer_link,manufacturer_value,model_id_link,model_id_value,model_number,monitor,name,operational_status,order_date,owned_by_link,owned_by_value,po_number,purchase_date,schedule_link,schedule_value,serial_number,short_description,skip_sync,start_date,subcategory,support_group_link,support_group_value,supported_by_link,supported_by_value,sys_class_name,sys_class_path,sys_created_by,sys_created_on,sys_domain_link,sys_domain_path,sys_domain_value,sys_mod_count,sys_updated_by,sys_updated_on,unverified,vendor_link,vendor_value,warranty_expiration
+46be1a35a9fe1981008a44a672b62e0d,false,2023-11-06 00:10:48.992000,,,,,,,,,,,false,,,,,,,false,,,,,,,,,,,,,,,,,,,,,,,,0,,,,,1,,,,,,,,,,,,,,,,,,,https://dev165675.service-now.com/api/now/v2/table/core_company/0e8b8e650a0a0b3b004f285ffbb1a4fc,0e8b8e650a0a0b3b004f285ffbb1a4fc,,,,false,Windows 2000 Hotfix - KB867282,1,,,,,,,,,,false,,,,,,,cmdb_ci_spkg,"""/!!/#,""",system,2006-02-07 22:59:57.000000,https://dev165675.service-now.com/api/now/v2/table/sys_user_group/global,/,global,15,system,2023-06-25 12:20:02.000000,false,,,

integration_tests/seeds/cmdb_ci_service_data.csv

@@ -0,0 +1,2 @@
+sys_id,_fivetran_deleted,_fivetran_synced,banner_image,banner_image_light,banner_text,city,contact_link,contact_value,coordinates_retrieved_on,country,customer,discount,fax_phone,fiscal_year,lat_long_error,latitude,longitude,manufacturer,market_cap,name,notes,num_employees,parent_link,parent_value,phone,primary,profits,publicly_traded,rank_tier,revenue_per_year,state,stock_price,stock_symbol,street,sys_class_name,sys_created_by,sys_created_on,sys_mod_count,sys_updated_by,sys_updated_on,theme_link,theme_value,vendor,vendor_manager,vendor_type,website,zip
+2f1c3218bfd101007a6d257b3f0739ee,false,2023-11-06 00:31:39,,,,,,,,USA,false,,,,,,,false,0,System manufacturer,,,,,,false,0,false,,0,,,,,core_company,admin,2013-05-08 20:23:29,0,admin,2013-05-08 20:23:29,,,false,,,,

integration_tests/seeds/core_company_data.csv

@@ -0,0 +1,2 @@
+sys_id,_fivetran_deleted,_fivetran_synced,banner_image,banner_image_light,banner_text,city,contact_link,contact_value,coordinates_retrieved_on,country,customer,discount,fax_phone,fiscal_year,lat_long_error,latitude,longitude,manufacturer,market_cap,name,notes,num_employees,parent_link,parent_value,phone,primary,profits,publicly_traded,rank_tier,revenue_per_year,state,stock_price,stock_symbol,street,sys_class_name,sys_created_by,sys_created_on,sys_mod_count,sys_updated_by,sys_updated_on,theme_link,theme_value,vendor,vendor_manager,vendor_type,website,zip
+2f1c3218bfd101007a6d257b3f0739ee,false,2023-11-06 00:31:39,,,,,,,,USA,false,,,,,,,false,0,System manufacturer,,,,,,false,0,false,,0,,,,,core_company,admin,2013-05-08 20:23:29,0,admin,2013-05-08 20:23:29,,,false,,,,

integration_tests/seeds/sys_user_group_data.csv

@@ -0,0 +1,2 @@
+sys_id,_fivetran_deleted,_fivetran_synced,active,cost_center_link,cost_center_value,default_assignee_link,default_assignee_value,description,email,exclude_manager,include_members,manager_link,manager_value,name,parent_link,parent_value,roles,source,sys_created_by,sys_created_on,sys_mod_count,sys_updated_by,sys_updated_on,type
+561cc5360f6e33004fbc309c4e767eb8,false,2023-11-06 03:50:57.165000,true,,,,,,,false,false,,,ATF Service Level Management  Group,,,,,admin,2019-07-13 16:00:26.000000,1,admin,2019-07-13 16:01:15.000000,

models/servicenow__activity_summary.sql

@@ -11,15 +11,15 @@ select
     impact,
     urgency,
     source_relation,
-    -- count(distinct case when is_task_active = true then task_id end) as total_active_tasks,
+    count(distinct case when is_task_active = true then task_id end) as total_active_tasks,
     count(distinct case when is_problem_task = true and is_task_active = true then task_id end) as total_active_problem_tasks,
     count(distinct case when is_change_task = true and is_task_active = true then task_id end) as total_active_change_tasks,
     count(distinct associated_problem_id) as total_associated_problems, 
     sum(problem_related_incidents) as total_incidents_caused_by_problems,
     count(distinct associated_change_request_id) as total_change_requests,
     count(distinct case when task_closed_at is not null then task_id end) as total_closed_tasks,
-    avg(task_minutes_open_to_close) as average_minutes_open_to_close
-    -- count(case when is_made_sla = true then 1 end) as total_tasks_made_slas
+    avg(task_minutes_open_to_close) as average_minutes_open_to_close,
+    count(case when is_made_sla = true then 1 end) as total_tasks_made_slas
 
 from ticket_enhanced
 group by 1,2,3,4,5,6