diff --git a/api-service/postman-collection/Obsrv API Service.postman_collection.json b/api-service/postman-collection/Obsrv API Service.postman_collection.json new file mode 100644 index 00000000..fb0c4a7f --- /dev/null +++ b/api-service/postman-collection/Obsrv API Service.postman_collection.json @@ -0,0 +1,546 @@ +{ + "info": { + "_postman_id": "a3107544-5d3b-4af9-8002-74ce81357719", + "name": "Obsrv API Service", + "schema": "https://schema.getpostman.com/json/collection/v2.0.0/collection.json", + "_exporter_id": "23908122", + "_collection_link": "https://restless-water-97719.postman.co/workspace/temp-workspace~5901a80f-1ada-4758-bf49-6f0fe63041c8/collection/23908122-a3107544-5d3b-4af9-8002-74ce81357719?action=share&source=collection_link&creator=23908122" + }, + "item": [ + { + "name": "Datasets", + "item": [ + { + "name": "List Datasets", + "request": { + "auth": { + "type": "noauth" + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"filters\": {\n \"status\": [\"Live\", \"Retired\"]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "{{HOST_IP}}/obsrv/v1/datasets/list" + }, + "response": [] + }, + { + "name": "Dataset Save", + "request": { + "auth": { + "type": "noauth" + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"sb-telemetry\",\n \"dataset_id\": \"sb-telemetry\",\n \"type\": \"dataset\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\",\n \"validation_mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\",\n \"dedup_period\": 1036800\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\",\n \"dedup_period\": 1036800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"number\"\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\"\n }\n }\n },\n \"mid\": {\n \"type\": \"string\",\n \"format\": \"uuid\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"format\": \"uuid\"\n },\n \"type\": {\n \"type\": \"string\"\n }\n }\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"type\": {\n \"type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"isRootOrg\": {\n \"type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\"\n },\n \"framework\": {\n \"type\": \"object\"\n },\n \"resourceType\": {\n \"type\": \"object\"\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\"\n }\n }\n },\n \"createdBy\": {\n \"type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"origin\": {\n \"type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n }\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\"\n }\n }\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"pageid\": {\n \"type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\"\n }\n }\n },\n \"state\": {\n \"type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"prevstate\": {\n \"type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\"\n }\n }\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"query\": {\n \"type\": \"string\"\n }\n }\n },\n \"mode\": {\n \"type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n }\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"format\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\"\n }\n }\n },\n \"env\": {\n \"type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\"\n }\n }\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"did\": {\n \"type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\"\n }\n }\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\"\n }\n }\n },\n \"version\": {\n \"type\": \"string\"\n }\n }\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n }\n },\n \"required\": [\n \"ets\",\n \"mid\"\n ]\n },\n \"denorm_config\": {\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"redis_db\": 4,\n \"denorm_out_field\": \"userdata\"\n }\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n },\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"published_date\": \"2023-07-03 00:00:00\",\n \"tags\": [],\n \"data_version\": 1\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST_IP}}/obsrv/v1/datasets/create?status=Live", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "obsrv", + "v1", + "datasets", + "create" + ], + "query": [ + { + "key": "status", + "value": "Live" + } + ] + } + }, + "response": [] + }, + { + "name": "Dataset Save Master", + "request": { + "auth": { + "type": "noauth" + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"sb-telemetry-user\",\n \"dataset_id\": \"sb-telemetry-user\",\n \"type\": \"master-dataset\",\n \"name\": \"sb-telemetry-user\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\",\n \"validation_mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\",\n \"dedup_period\": 1036800\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\",\n \"dedup_period\": 1036800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\"\n },\n \"grade\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n },\n \"userlogintype\": {\n \"type\": \"string\"\n },\n \"usersignintype\": {\n \"type\": \"string\"\n },\n \"usertype\": {\n \"type\": \"string\"\n },\n \"language\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"subject\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"district\": {\n \"type\": \"string\"\n },\n \"state\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"id\"\n ]\n },\n \"denorm_config\": {\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": []\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry-user\"\n },\n \"dataset_config\": {\n \"data_key\": \"id\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.masterdata.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": false,\n \"redis_db\": 4\n },\n \"status\": \"Live\",\n \"tags\": [],\n \"data_version\": null,\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2023-07-12T06:07:10.268Z\",\n \"updated_date\": \"2023-07-12T06:07:10.160Z\",\n \"published_date\": \"2023-07-01T00:00:00.000Z\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "{{HOST_IP}}/obsrv/v1/datasets/create" + }, + "response": [] + }, + { + "name": "Update Dataset", + "request": { + "auth": { + "type": "noauth" + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"sb-telemetry\",\n \"dataset_id\": \"sb-telemetry\",\n \"type\": \"dataset1\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\",\n \"validation_mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\",\n \"dedup_period\": 1036800\n },\n \"batch_id\": \"id\"\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\",\n \"dedup_period\": 1036800\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"syncts\": {\n \"type\": \"integer\"\n },\n \"ets\": {\n \"type\": \"number\"\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\"\n },\n \"pp_validation_processed\": {\n \"type\": \"boolean\"\n },\n \"pp_duplicate_skipped\": {\n \"type\": \"boolean\"\n },\n \"user_denorm\": {\n \"type\": \"boolean\"\n },\n \"device_denorm\": {\n \"type\": \"boolean\"\n },\n \"loc_denorm\": {\n \"type\": \"boolean\"\n },\n \"content_denorm\": {\n \"type\": \"boolean\"\n },\n \"coll_denorm\": {\n \"type\": \"boolean\"\n }\n }\n },\n \"mid\": {\n \"type\": \"string\",\n \"format\": \"uuid\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"format\": \"uuid\"\n },\n \"type\": {\n \"type\": \"string\"\n }\n }\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"visits\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"type\": {\n \"type\": \"string\"\n },\n \"duration\": {\n \"type\": \"number\"\n },\n \"size\": {\n \"type\": \"integer\"\n },\n \"query\": {\n \"type\": \"string\"\n },\n \"filters\": {\n \"type\": \"object\",\n \"properties\": {\n \"objectType\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"version\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"status\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"id\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"isRootOrg\": {\n \"type\": \"boolean\"\n },\n \"trackable.enabled\": {\n \"type\": \"string\"\n },\n \"channel\": {\n \"type\": \"object\"\n },\n \"framework\": {\n \"type\": \"object\"\n },\n \"resourceType\": {\n \"type\": \"object\"\n },\n \"identifier\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"contentType\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"mimeType\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"hashTagId\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"compatibilityLevel\": {\n \"type\": \"object\",\n \"properties\": {\n \"min\": {\n \"type\": \"integer\"\n },\n \"max\": {\n \"type\": \"integer\"\n }\n }\n },\n \"createdBy\": {\n \"type\": \"string\"\n },\n \"mediaType\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"origin\": {\n \"type\": \"string\"\n },\n \"primaryCategory\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"trackable\": {\n \"enabled\": {\n \"type\": \"string\"\n }\n }\n }\n },\n \"sort\": {\n \"type\": \"object\",\n \"properties\": {\n \"lastUpdatedOn\": {\n \"type\": \"string\"\n }\n }\n },\n \"topn\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"pageid\": {\n \"type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\"\n },\n \"data\": {\n \"type\": \"string\"\n },\n \"uaspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"agent\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"system\": {\n \"type\": \"string\"\n },\n \"platform\": {\n \"type\": \"string\"\n },\n \"raw\": {\n \"type\": \"string\"\n }\n }\n },\n \"state\": {\n \"type\": \"string\"\n },\n \"props\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"prevstate\": {\n \"type\": \"string\"\n },\n \"dspec\": {\n \"type\": \"object\",\n \"properties\": {\n \"os\": {\n \"type\": \"string\"\n },\n \"make\": {\n \"type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\"\n },\n \"idisk\": {\n \"type\": \"number\"\n },\n \"edisk\": {\n \"type\": \"number\"\n },\n \"scrn\": {\n \"type\": \"number\"\n },\n \"camera\": {\n \"type\": \"string\"\n },\n \"cpu\": {\n \"type\": \"string\"\n },\n \"sims\": {\n \"type\": \"integer\"\n },\n \"webview\": {\n \"type\": \"string\"\n }\n }\n },\n \"extra\": {\n \"type\": \"object\",\n \"properties\": {\n \"pos\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"values\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"query\": {\n \"type\": \"string\"\n }\n }\n },\n \"mode\": {\n \"type\": \"string\"\n }\n },\n \"duration\": {\n \"type\": \"string\"\n }\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"format\": \"date-time\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\"\n }\n }\n },\n \"env\": {\n \"type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\"\n },\n \"l2\": {\n \"type\": \"string\"\n },\n \"l3\": {\n \"type\": \"string\"\n }\n }\n },\n \"cdata\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n },\n \"did\": {\n \"type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\"\n }\n }\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\"\n }\n }\n },\n \"version\": {\n \"type\": \"string\"\n }\n }\n },\n \"tags\": {\n \"type\": \"array\",\n \"additionalProperties\": true\n }\n },\n \"required\": [\n \"ets\",\n \"mid\"\n ]\n },\n \"denorm_config\": {\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"denorm_fields\": [\n {\n \"denorm_key\": \"actor.id\",\n \"redis_db\": 4,\n \"denorm_out_field\": \"userdata\"\n }\n ]\n },\n \"router_config\": {\n \"topic\": \"sb-telemetry\"\n },\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n },\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"published_date\": \"2023-07-03 00:00:00\",\n \"tags\": [],\n \"data_version\": 1\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "{{HOST_IP}}/obsrv/v1/datasets/update" + }, + "response": [] + }, + { + "name": "Dataset Read", + "request": { + "auth": { + "type": "noauth" + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{HOST_IP}}/obsrv/v1/datasets/get/?status=Live", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "obsrv", + "v1", + "datasets", + "get", + "" + ], + "query": [ + { + "key": "status", + "value": "Live" + } + ] + } + }, + "response": [] + } + ] + }, + { + "name": "Datasources", + "item": [ + { + "name": "List Datasources", + "request": { + "auth": { + "type": "noauth" + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"filters\": {\n \"status\": [\n \"Live\",\n \"Retired\"\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "{{HOST_IP}}/obsrv/v1/datasources/list" + }, + "response": [] + }, + { + "name": "Datasource Save", + "request": { + "auth": { + "type": "noauth" + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"sb-telemetry_sb-telemetry\",\n \"datasource\": \"sb-telemetry\",\n \"dataset_id\": \"sb-telemetry\",\n \"ingestion_spec\": {\n \"type\": \"kafka\",\n \"spec\": {\n \"dataSchema\": {\n \"dataSource\": \"sb-telemetry.1_DAY\",\n \"dimensionsSpec\": {\n \"dimensions\": [\n {\n \"type\": \"string\",\n \"name\": \"eid\"\n },\n {\n \"type\": \"string\",\n \"name\": \"ver\"\n },\n {\n \"type\": \"long\",\n \"name\": \"syncts\"\n },\n {\n \"type\": \"boolean\",\n \"name\": \"flags_ex_processed\"\n },\n {\n \"type\": \"boolean\",\n \"name\": \"flags_pp_validation_processed\"\n },\n {\n \"type\": \"boolean\",\n \"name\": \"flags_pp_duplicate_skipped\"\n },\n {\n \"type\": \"boolean\",\n \"name\": \"flags_user_denorm\"\n },\n {\n \"type\": \"boolean\",\n \"name\": \"flags_device_denorm\"\n },\n {\n \"type\": \"boolean\",\n \"name\": \"flags_loc_denorm\"\n },\n {\n \"type\": \"boolean\",\n \"name\": \"flags_content_denorm\"\n },\n {\n \"type\": \"boolean\",\n \"name\": \"flags_coll_denorm\"\n },\n {\n \"type\": \"string\",\n \"name\": \"mid\"\n },\n {\n \"type\": \"string\",\n \"name\": \"actor_id\"\n },\n {\n \"type\": \"string\",\n \"name\": \"actor_type\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_type\"\n },\n {\n \"type\": \"long\",\n \"name\": \"edata_duration\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_query\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_objectType\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_version\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_status\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_id\"\n },\n {\n \"type\": \"boolean\",\n \"name\": \"edata_filters_isRootOrg\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_filters_trackable_enabled\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_identifier\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_contentType\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_mimeType\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_hashTagId\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_filters_createdBy\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_mediaType\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_filters_origin\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_primaryCategory\"\n },\n {\n \"name\": \"edata_filters_trackable\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_sort_lastUpdatedOn\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_topn\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_pageid\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_uri\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_subtype\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_id\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_data\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_uaspec_agent\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_uaspec_ver\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_uaspec_system\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_uaspec_platform\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_uaspec_raw\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_state\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_props\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_prevstate\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_dspec_os\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_dspec_make\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_dspec_id\"\n },\n {\n \"type\": \"long\",\n \"name\": \"edata_dspec_idisk\"\n },\n {\n \"type\": \"long\",\n \"name\": \"edata_dspec_edisk\"\n },\n {\n \"type\": \"long\",\n \"name\": \"edata_dspec_scrn\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_dspec_camera\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_dspec_cpu\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_dspec_webview\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_extra_pos\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_extra_values\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_extra_query\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_mode\"\n },\n {\n \"type\": \"string\",\n \"name\": \"@timestamp\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_channel\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_pdata_id\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_pdata_ver\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_pdata_pid\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_env\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_sid\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_rollup_l1\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_rollup_l2\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_rollup_l3\"\n },\n {\n \"type\": \"array\",\n \"name\": \"context_cdata\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_did\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_uid\"\n },\n {\n \"type\": \"string\",\n \"name\": \"object_id\"\n },\n {\n \"type\": \"string\",\n \"name\": \"object_type\"\n },\n {\n \"type\": \"string\",\n \"name\": \"object_ver\"\n },\n {\n \"type\": \"string\",\n \"name\": \"object_rollup_l1\"\n },\n {\n \"type\": \"string\",\n \"name\": \"object_version\"\n },\n {\n \"type\": \"array\",\n \"name\": \"tags\"\n } \n ]\n },\n \"timestampSpec\": {\n \"column\": \"ets\",\n \"format\": \"auto\"\n },\n \"metricsSpec\": [\n {\n \"type\": \"doubleSum\",\n \"name\": \"edata_size\",\n \"fieldName\": \"edata_size\"\n },\n {\n \"type\": \"doubleSum\",\n \"name\": \"edata_filters_compatibilityLevel_min\",\n \"fieldName\": \"edata_filters_compatibilityLevel_min\"\n },\n {\n \"type\": \"doubleSum\",\n \"name\": \"edata_filters_compatibilityLevel_max\",\n \"fieldName\": \"edata_filters_compatibilityLevel_max\"\n },\n {\n \"type\": \"doubleSum\",\n \"name\": \"edata_dspec_sims\",\n \"fieldName\": \"edata_dspec_sims\"\n }\n ],\n \"granularitySpec\": {\n \"type\": \"uniform\",\n \"segmentGranularity\": \"DAY\",\n \"rollup\": false\n }\n },\n \"tuningConfig\": {\n \"type\": \"kafka\",\n \"maxBytesInMemory\": 134217728,\n \"maxRowsPerSegment\": 500000,\n \"logParseExceptions\": true\n },\n \"ioConfig\": {\n \"type\": \"kafka\",\n \"topic\": \"sb-telemetry\",\n \"consumerProperties\": {\n \"bootstrap.servers\": \"kafka-headless.kafka.svc:9092\"\n },\n \"taskCount\": 1,\n \"replicas\": 1,\n \"taskDuration\": \"PT1H\",\n \"useEarliestOffset\": true,\n \"completionTimeout\": \"PT1H\",\n \"inputFormat\": {\n \"type\": \"json\",\n \"flattenSpec\": {\n \"useFieldDiscovery\": true,\n \"fields\": [\n {\n \"type\": \"path\",\n \"expr\": \"$.eid\",\n \"name\": \"eid\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.ver\",\n \"name\": \"ver\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.syncts\",\n \"name\": \"syncts\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.ets\",\n \"name\": \"ets\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.flags.ex_processed\",\n \"name\": \"flags_ex_processed\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.flags.pp_validation_processed\",\n \"name\": \"flags_pp_validation_processed\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.flags.pp_duplicate_skipped\",\n \"name\": \"flags_pp_duplicate_skipped\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.flags.user_denorm\",\n \"name\": \"flags_user_denorm\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.flags.device_denorm\",\n \"name\": \"flags_device_denorm\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.flags.loc_denorm\",\n \"name\": \"flags_loc_denorm\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.flags.content_denorm\",\n \"name\": \"flags_content_denorm\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.flags.coll_denorm\",\n \"name\": \"flags_coll_denorm\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.mid\",\n \"name\": \"mid\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.actor.id\",\n \"name\": \"actor_id\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.actor.type\",\n \"name\": \"actor_type\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.type\",\n \"name\": \"edata_type\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.duration\",\n \"name\": \"edata_duration\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.query\",\n \"name\": \"edata_query\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.objectType[*]\",\n \"name\": \"edata_filters_objectType\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.version[*]\",\n \"name\": \"edata_filters_version\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.status[*]\",\n \"name\": \"edata_filters_status\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.id[*]\",\n \"name\": \"edata_filters_id\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.isRootOrg\",\n \"name\": \"edata_filters_isRootOrg\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.trackable.enabled\",\n \"name\": \"edata_filters_trackable_enabled\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.identifier[*]\",\n \"name\": \"edata_filters_identifier\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.contentType[*]\",\n \"name\": \"edata_filters_contentType\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.mimeType[*]\",\n \"name\": \"edata_filters_mimeType\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.hashTagId[*]\",\n \"name\": \"edata_filters_hashTagId\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.createdBy\",\n \"name\": \"edata_filters_createdBy\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.mediaType[*]\",\n \"name\": \"edata_filters_mediaType\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.origin\",\n \"name\": \"edata_filters_origin\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.primaryCategory[*]\",\n \"name\": \"edata_filters_primaryCategory\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.trackable\",\n \"name\": \"edata_filters_trackable\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.sort.lastUpdatedOn\",\n \"name\": \"edata_sort_lastUpdatedOn\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.topn[*]\",\n \"name\": \"edata_topn\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.pageid\",\n \"name\": \"edata_pageid\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.uri\",\n \"name\": \"edata_uri\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.subtype\",\n \"name\": \"edata_subtype\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.id\",\n \"name\": \"edata_id\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.data\",\n \"name\": \"edata_data\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.uaspec.agent\",\n \"name\": \"edata_uaspec_agent\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.uaspec.ver\",\n \"name\": \"edata_uaspec_ver\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.uaspec.system\",\n \"name\": \"edata_uaspec_system\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.uaspec.platform\",\n \"name\": \"edata_uaspec_platform\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.uaspec.raw\",\n \"name\": \"edata_uaspec_raw\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.state\",\n \"name\": \"edata_state\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.props[*]\",\n \"name\": \"edata_props\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.prevstate\",\n \"name\": \"edata_prevstate\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.os\",\n \"name\": \"edata_dspec_os\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.make\",\n \"name\": \"edata_dspec_make\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.id\",\n \"name\": \"edata_dspec_id\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.idisk\",\n \"name\": \"edata_dspec_idisk\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.edisk\",\n \"name\": \"edata_dspec_edisk\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.scrn\",\n \"name\": \"edata_dspec_scrn\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.camera\",\n \"name\": \"edata_dspec_camera\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.cpu\",\n \"name\": \"edata_dspec_cpu\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.webview\",\n \"name\": \"edata_dspec_webview\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.extra.pos[*]\",\n \"name\": \"edata_extra_pos\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.extra.values[*]\",\n \"name\": \"edata_extra_values\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.extra.query\",\n \"name\": \"edata_extra_query\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.mode\",\n \"name\": \"edata_mode\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.@timestamp\",\n \"name\": \"@timestamp\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.channel\",\n \"name\": \"context_channel\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.pdata.id\",\n \"name\": \"context_pdata_id\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.pdata.ver\",\n \"name\": \"context_pdata_ver\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.pdata.pid\",\n \"name\": \"context_pdata_pid\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.env\",\n \"name\": \"context_env\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.sid\",\n \"name\": \"context_sid\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.rollup.l1\",\n \"name\": \"context_rollup_l1\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.rollup.l2\",\n \"name\": \"context_rollup_l2\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.rollup.l3\",\n \"name\": \"context_rollup_l3\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.cdata[*]\",\n \"name\": \"context_cdata\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.did\",\n \"name\": \"context_did\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.uid\",\n \"name\": \"context_uid\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.object.id\",\n \"name\": \"object_id\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.object.type\",\n \"name\": \"object_type\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.object.ver\",\n \"name\": \"object_ver\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.object.rollup.l1\",\n \"name\": \"object_rollup_l1\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.object.version\",\n \"name\": \"object_version\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.tags[*]\",\n \"name\": \"tags\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.size\",\n \"name\": \"edata_size\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.compatibilityLevel.min\",\n \"name\": \"edata_filters_compatibilityLevel_min\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.compatibilityLevel.max\",\n \"name\": \"edata_filters_compatibilityLevel_max\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.sims\",\n \"name\": \"edata_dspec_sims\"\n }\n ]\n }\n },\n \"appendToExisting\": false\n }\n }\n },\n \"datasource_ref\": \"sb-telemetry.1_DAY\",\n \"retention_period\": {\n \"enabled\": \"false\"\n },\n \"archival_policy\": {\n \"enabled\": \"false\"\n },\n \"purge_policy\": {\n \"enabled\": \"false\"\n },\n \"backup_config\": {\n \"enabled\": \"false\"\n },\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"published_date\": \"2023-07-03 00:00:00\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "{{HOST_IP}}/obsrv/v1/datasources/create" + }, + "response": [] + }, + { + "name": "Datasource Update", + "request": { + "auth": { + "type": "noauth" + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"sb-telemetry_sb-telemetry\",\n \"datasource\": \"sb-telemetry\",\n \"dataset_id\": \"sb-telemetry\",\n \"ingestion_spec\": {\n \"type\": \"kafka\",\n \"spec\": {\n \"dataSchema\": {\n \"dataSource\": \"sb-telemetry.1_DAY\",\n \"dimensionsSpec\": {\n \"dimensions\": [\n {\n \"type\": \"string\",\n \"name\": \"eid\"\n },\n {\n \"type\": \"string\",\n \"name\": \"ver\"\n },\n {\n \"type\": \"long\",\n \"name\": \"syncts\"\n },\n {\n \"type\": \"boolean\",\n \"name\": \"flags_ex_processed\"\n },\n {\n \"type\": \"boolean\",\n \"name\": \"flags_pp_validation_processed\"\n },\n {\n \"type\": \"boolean\",\n \"name\": \"flags_pp_duplicate_skipped\"\n },\n {\n \"type\": \"boolean\",\n \"name\": \"flags_user_denorm\"\n },\n {\n \"type\": \"boolean\",\n \"name\": \"flags_device_denorm\"\n },\n {\n \"type\": \"boolean\",\n \"name\": \"flags_loc_denorm\"\n },\n {\n \"type\": \"boolean\",\n \"name\": \"flags_content_denorm\"\n },\n {\n \"type\": \"boolean\",\n \"name\": \"flags_coll_denorm\"\n },\n {\n \"type\": \"string\",\n \"name\": \"mid\"\n },\n {\n \"type\": \"string\",\n \"name\": \"actor_id\"\n },\n {\n \"type\": \"string\",\n \"name\": \"actor_type\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_type\"\n },\n {\n \"type\": \"long\",\n \"name\": \"edata_duration\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_query\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_objectType\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_version\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_status\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_id\"\n },\n {\n \"type\": \"boolean\",\n \"name\": \"edata_filters_isRootOrg\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_filters_trackable_enabled\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_identifier\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_contentType\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_mimeType\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_hashTagId\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_filters_createdBy\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_mediaType\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_filters_origin\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_primaryCategory\"\n },\n {\n \"name\": \"edata_filters_trackable\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_sort_lastUpdatedOn\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_topn\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_pageid\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_uri\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_subtype\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_id\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_data\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_uaspec_agent\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_uaspec_ver\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_uaspec_system\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_uaspec_platform\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_uaspec_raw\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_state\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_props\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_prevstate\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_dspec_os\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_dspec_make\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_dspec_id\"\n },\n {\n \"type\": \"long\",\n \"name\": \"edata_dspec_idisk\"\n },\n {\n \"type\": \"long\",\n \"name\": \"edata_dspec_edisk\"\n },\n {\n \"type\": \"long\",\n \"name\": \"edata_dspec_scrn\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_dspec_camera\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_dspec_cpu\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_dspec_webview\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_extra_pos\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_extra_values\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_extra_query\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_mode\"\n },\n {\n \"type\": \"string\",\n \"name\": \"@timestamp\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_channel\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_pdata_id\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_pdata_ver\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_pdata_pid\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_env\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_sid\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_rollup_l1\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_rollup_l2\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_rollup_l3\"\n },\n {\n \"type\": \"array\",\n \"name\": \"context_cdata\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_did\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_uid\"\n },\n {\n \"type\": \"string\",\n \"name\": \"object_id\"\n },\n {\n \"type\": \"string\",\n \"name\": \"object_type\"\n },\n {\n \"type\": \"string\",\n \"name\": \"object_ver\"\n },\n {\n \"type\": \"string\",\n \"name\": \"object_rollup_l1\"\n },\n {\n \"type\": \"string\",\n \"name\": \"object_version\"\n },\n {\n \"type\": \"array\",\n \"name\": \"tags\"\n } \n ]\n },\n \"timestampSpec\": {\n \"column\": \"ets\",\n \"format\": \"auto\"\n },\n \"metricsSpec\": [\n {\n \"type\": \"doubleSum\",\n \"name\": \"edata_size\",\n \"fieldName\": \"edata_size\"\n },\n {\n \"type\": \"doubleSum\",\n \"name\": \"edata_filters_compatibilityLevel_min\",\n \"fieldName\": \"edata_filters_compatibilityLevel_min\"\n },\n {\n \"type\": \"doubleSum\",\n \"name\": \"edata_filters_compatibilityLevel_max\",\n \"fieldName\": \"edata_filters_compatibilityLevel_max\"\n },\n {\n \"type\": \"doubleSum\",\n \"name\": \"edata_dspec_sims\",\n \"fieldName\": \"edata_dspec_sims\"\n }\n ],\n \"granularitySpec\": {\n \"type\": \"uniform\",\n \"segmentGranularity\": \"DAY\",\n \"rollup\": false\n }\n },\n \"tuningConfig\": {\n \"type\": \"kafka\",\n \"maxBytesInMemory\": 134217728,\n \"maxRowsPerSegment\": 500000,\n \"logParseExceptions\": true\n },\n \"ioConfig\": {\n \"type\": \"kafka\",\n \"topic\": \"sb-telemetry\",\n \"consumerProperties\": {\n \"bootstrap.servers\": \"kafka-headless.kafka.svc:9092\"\n },\n \"taskCount\": 1,\n \"replicas\": 1,\n \"taskDuration\": \"PT1H\",\n \"useEarliestOffset\": true,\n \"completionTimeout\": \"PT1H\",\n \"inputFormat\": {\n \"type\": \"json\",\n \"flattenSpec\": {\n \"useFieldDiscovery\": true,\n \"fields\": [\n {\n \"type\": \"path\",\n \"expr\": \"$.eid\",\n \"name\": \"eid\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.ver\",\n \"name\": \"ver\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.syncts\",\n \"name\": \"syncts\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.ets\",\n \"name\": \"ets\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.flags.ex_processed\",\n \"name\": \"flags_ex_processed\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.flags.pp_validation_processed\",\n \"name\": \"flags_pp_validation_processed\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.flags.pp_duplicate_skipped\",\n \"name\": \"flags_pp_duplicate_skipped\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.flags.user_denorm\",\n \"name\": \"flags_user_denorm\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.flags.device_denorm\",\n \"name\": \"flags_device_denorm\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.flags.loc_denorm\",\n \"name\": \"flags_loc_denorm\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.flags.content_denorm\",\n \"name\": \"flags_content_denorm\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.flags.coll_denorm\",\n \"name\": \"flags_coll_denorm\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.mid\",\n \"name\": \"mid\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.actor.id\",\n \"name\": \"actor_id\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.actor.type\",\n \"name\": \"actor_type\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.type\",\n \"name\": \"edata_type\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.duration\",\n \"name\": \"edata_duration\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.query\",\n \"name\": \"edata_query\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.objectType[*]\",\n \"name\": \"edata_filters_objectType\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.version[*]\",\n \"name\": \"edata_filters_version\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.status[*]\",\n \"name\": \"edata_filters_status\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.id[*]\",\n \"name\": \"edata_filters_id\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.isRootOrg\",\n \"name\": \"edata_filters_isRootOrg\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.trackable.enabled\",\n \"name\": \"edata_filters_trackable_enabled\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.identifier[*]\",\n \"name\": \"edata_filters_identifier\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.contentType[*]\",\n \"name\": \"edata_filters_contentType\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.mimeType[*]\",\n \"name\": \"edata_filters_mimeType\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.hashTagId[*]\",\n \"name\": \"edata_filters_hashTagId\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.createdBy\",\n \"name\": \"edata_filters_createdBy\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.mediaType[*]\",\n \"name\": \"edata_filters_mediaType\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.origin\",\n \"name\": \"edata_filters_origin\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.primaryCategory[*]\",\n \"name\": \"edata_filters_primaryCategory\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.trackable\",\n \"name\": \"edata_filters_trackable\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.sort.lastUpdatedOn\",\n \"name\": \"edata_sort_lastUpdatedOn\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.topn[*]\",\n \"name\": \"edata_topn\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.pageid\",\n \"name\": \"edata_pageid\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.uri\",\n \"name\": \"edata_uri\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.subtype\",\n \"name\": \"edata_subtype\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.id\",\n \"name\": \"edata_id\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.data\",\n \"name\": \"edata_data\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.uaspec.agent\",\n \"name\": \"edata_uaspec_agent\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.uaspec.ver\",\n \"name\": \"edata_uaspec_ver\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.uaspec.system\",\n \"name\": \"edata_uaspec_system\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.uaspec.platform\",\n \"name\": \"edata_uaspec_platform\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.uaspec.raw\",\n \"name\": \"edata_uaspec_raw\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.state\",\n \"name\": \"edata_state\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.props[*]\",\n \"name\": \"edata_props\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.prevstate\",\n \"name\": \"edata_prevstate\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.os\",\n \"name\": \"edata_dspec_os\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.make\",\n \"name\": \"edata_dspec_make\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.id\",\n \"name\": \"edata_dspec_id\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.idisk\",\n \"name\": \"edata_dspec_idisk\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.edisk\",\n \"name\": \"edata_dspec_edisk\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.scrn\",\n \"name\": \"edata_dspec_scrn\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.camera\",\n \"name\": \"edata_dspec_camera\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.cpu\",\n \"name\": \"edata_dspec_cpu\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.webview\",\n \"name\": \"edata_dspec_webview\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.extra.pos[*]\",\n \"name\": \"edata_extra_pos\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.extra.values[*]\",\n \"name\": \"edata_extra_values\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.extra.query\",\n \"name\": \"edata_extra_query\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.mode\",\n \"name\": \"edata_mode\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.@timestamp\",\n \"name\": \"@timestamp\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.channel\",\n \"name\": \"context_channel\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.pdata.id\",\n \"name\": \"context_pdata_id\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.pdata.ver\",\n \"name\": \"context_pdata_ver\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.pdata.pid\",\n \"name\": \"context_pdata_pid\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.env\",\n \"name\": \"context_env\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.sid\",\n \"name\": \"context_sid\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.rollup.l1\",\n \"name\": \"context_rollup_l1\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.rollup.l2\",\n \"name\": \"context_rollup_l2\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.rollup.l3\",\n \"name\": \"context_rollup_l3\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.cdata[*]\",\n \"name\": \"context_cdata\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.did\",\n \"name\": \"context_did\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.uid\",\n \"name\": \"context_uid\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.object.id\",\n \"name\": \"object_id\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.object.type\",\n \"name\": \"object_type\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.object.ver\",\n \"name\": \"object_ver\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.object.rollup.l1\",\n \"name\": \"object_rollup_l1\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.object.version\",\n \"name\": \"object_version\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.tags[*]\",\n \"name\": \"tags\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.size\",\n \"name\": \"edata_size\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.compatibilityLevel.min\",\n \"name\": \"edata_filters_compatibilityLevel_min\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.compatibilityLevel.max\",\n \"name\": \"edata_filters_compatibilityLevel_max\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.sims\",\n \"name\": \"edata_dspec_sims\"\n }\n ]\n }\n },\n \"appendToExisting\": false\n }\n }\n },\n \"datasource_ref\": \"sb-telemetry.1_DAY\",\n \"retention_period\": {\n \"enabled\": \"false\"\n },\n \"archival_policy\": {\n \"enabled\": \"false\"\n },\n \"purge_policy\": {\n \"enabled\": \"false\"\n },\n \"backup_config\": {\n \"enabled\": \"false\"\n },\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"published_date\": \"2023-07-03 00:00:00\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "{{HOST_IP}}/obsrv/v1/datasources/update" + }, + "response": [] + }, + { + "name": "Datasource Read", + "request": { + "auth": { + "type": "noauth" + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{HOST_IP}}/obsrv/v1/datasources/get/sb-telemetry_sb-telemetry?status=Live", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "obsrv", + "v1", + "datasources", + "get", + "sb-telemetry_sb-telemetry" + ], + "query": [ + { + "key": "status", + "value": "Live" + } + ] + } + }, + "response": [] + } + ] + }, + { + "name": "Source Config", + "item": [ + { + "name": "List Source Configs", + "request": { + "auth": { + "type": "noauth" + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"filters\": {\n \"status\": [\n \"Live\",\n \"Retired\"\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "{{HOST_IP}}/obsrv/v1/datasets/source/config/list" + }, + "response": [] + }, + { + "name": "Dataset source config read", + "protocolProfileBehavior": { + "disableBodyPruning": true + }, + "request": { + "auth": { + "type": "noauth" + }, + "method": "GET", + "header": [], + "body": { + "mode": "raw", + "raw": "", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{HOST_IP}}/obsrv/v1/datasets/source/config/get/6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0?status=Live", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "obsrv", + "v1", + "datasets", + "source", + "config", + "get", + "6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0" + ], + "query": [ + { + "key": "status", + "value": "Live" + } + ] + } + }, + "response": [] + }, + { + "name": "Dataset source config save", + "request": { + "auth": { + "type": "noauth" + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0\",\n \"dataset_id\": \"sb-telemetry\",\n \"connector_type\": \"kafka\",\n \"connector_config\": {\n \"type\": \"kafka\",\n \"topic\": \"telemetry.ingest\",\n \"kafkaBrokers\": \"kafka-headless.kafka.svc:9092\"\n },\n \"status\": \"Live\",\n \"published_date\": \"2023-03-24 12:19:32.091544\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "{{HOST_IP}}/obsrv/v1/datasets/source/config/create" + }, + "response": [] + }, + { + "name": "Dataset source config update", + "request": { + "auth": { + "type": "noauth" + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0\",\n \"dataset_id\": \"sb-telemetry\",\n \"connector_type\": \"kafka\",\n \"connector_config\": {\n \"type\": \"kafka\",\n \"topic\": \"telemetry.ingest\",\n \"kafkaBrokers\": \"kafka-headless.kafka.svc:9092\"\n },\n \"status\": \"Live\",\n \"published_date\": \"2023-03-24 12:19:32.091544\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "{{HOST_IP}}/obsrv/v1/datasets/source/config/update" + }, + "response": [] + } + ] + }, + { + "name": "Query (Data OUT APIs)", + "item": [ + { + "name": "native query request", + "request": { + "auth": { + "type": "noauth" + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"dataSource\": \"dataset-1\"\n },\n \"query\": {\n \"queryType\": \"timeseries\",\n \"dataSource\": \"dataset-1\",\n \"intervals\": {\n \"type\": \"intervals\",\n \"intervals\": [\n \"2023-06-15/2023-07-11\"\n ]\n },\n \"granularity\": \"week\",\n \"aggregations\": [\n {\n \"type\": \"count\",\n \"name\": \"total_records\"\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "{{HOST_IP}}/obsrv/v1/data/query/dataset-1_dataset-1.1_DAY" + }, + "response": [] + }, + { + "name": "sql query request", + "event": [ + { + "listen": "prerequest", + "script": { + "exec": [ + " " + ], + "type": "text/javascript" + } + } + ], + "request": { + "auth": { + "type": "noauth" + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"context\": {\n \"dataSource\": \"dataset-1\"\n },\n \"querySql\": {\n \"query\": \"SELECT COUNT(*) AS totalRatingsCount FROM \\\"dataset-1\\\" LIMIT 100\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "{{HOST_IP}}/obsrv/v1/data/sql-query/dataset-1_dataset-1.1_DA" + }, + "response": [] + } + ] + }, + { + "name": "Data ingest", + "item": [ + { + "name": "Data Ingest Batch", + "request": { + "auth": { + "type": "noauth" + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"data\": {\n \"id\": \"batch0017\",\n \"events\": [\n {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682,\n \"flags\": {\n \"ex_processed\": true,\n \"pp_validation_processed\": true,\n \"pp_duplicate_skipped\": true,\n \"device_denorm\": true,\n \"user_denorm\": true,\n \"loc_denorm\": true\n },\n \"derivedlocationdata\": {\n \"district\": \"AGRA\",\n \"from\": \"user-profile\",\n \"state\": \"Uttar Pradesh\"\n },\n \"mid\": \"6c3fc8d2-357e-479b-b0c0-afdde6e5c6c0\",\n \"type\": \"events\",\n \"actor\": {\n \"type\": \"User\",\n \"id\": \"b6d0a460-430e-4077-8c50-dfe38e9aade3\"\n },\n \"edata\": {\n \"type\": \"session\"\n },\n \"userdata\": {\n \"subject\": [],\n \"district\": \"AGRA\",\n \"usersubtype\": \"hm\",\n \"grade\": [],\n \"usersignintype\": \"Self-Signed-In\",\n \"usertype\": \"administrator\",\n \"userlogintype\": \"administrator\",\n \"state\": \"Uttar Pradesh\"\n },\n \"@timestamp\": \"2022-11-16T09:45:49.682Z\",\n \"devicedata\": {\n \"statecustomcode\": \"29\",\n \"country\": \"India\",\n \"iso3166statecode\": \"IN-KA\",\n \"city\": \"Bengaluru\",\n \"countrycode\": \"IN\",\n \"state\": \"Karnataka\",\n \"devicespec\": {\n \"idisk\": \"106.47\",\n \"webview\": \"107.0.5304.105\",\n \"os\": \"Android 12\",\n \"scrn\": \"6.53\",\n \"sims\": \"-1\",\n \"cpu\": \"abi: arm64-v8a processor\\t: 0 \",\n \"id\": \"ac4ad4ac3feda0f2b17835b81e736c88c194dc89\",\n \"camera\": \"\",\n \"edisk\": \"106.27\",\n \"make\": \"vivo 1915\"\n },\n \"statecode\": \"KA\",\n \"firstaccess\": 1660038763481,\n \"districtcustom\": \"BENGALURU URBAN SOUTH\",\n \"statecustomname\": \"Karnataka\",\n \"userdeclared\": {\n \"district\": \"AGRA\",\n \"state\": \"Uttar Pradesh\"\n }\n },\n \"context\": {\n \"cdata\": [\n {\n \"id\": \"a3c784f0-61d8-43e4-a92a-373fd4338c1d\",\n \"type\": \"UserSession\"\n }\n ],\n \"env\": \"sdk\",\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"preprod.diksha.app\",\n \"pid\": \"sunbird.app\",\n \"ver\": \"4.10.1023preproduction\"\n },\n \"sid\": \"a3c784f0-61d8-43e4-a92a-373fd4338c1d\",\n \"did\": \"ac4ad4ac3feda0f2b17835b81e736c88c194dc89\",\n \"rollup\": {\n \"l1\": \"0126796199493140480\"\n }\n },\n \"object\": {\n \"id\": \"\",\n \"type\": \"\",\n \"version\": \"\",\n \"rollup\": {}\n }\n },\n {\n \"eid\": \"INTERACT\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1.668591793792E12,\n \"flags\": {\n \"ex_processed\": true,\n \"pp_validation_processed\": true,\n \"pp_duplicate_skipped\": true,\n \"device_denorm\": true,\n \"user_denorm\": true,\n \"loc_denorm\": true\n },\n \"derivedlocationdata\": {\n \"district\": \"AGRA\",\n \"from\": \"user-profile\",\n \"state\": \"Uttar Pradesh\"\n },\n \"mid\": \"0fd4d021-1372-4b0b-980c-29d167f77c9a\",\n \"type\": \"events\",\n \"actor\": {\n \"type\": \"User\",\n \"id\": \"b6d0a460-430e-4077-8c50-dfe38e9aade3\"\n },\n \"edata\": {\n \"type\": \"session\",\n \"duration\": 20126.0\n },\n \"userdata\": {\n \"subject\": [],\n \"district\": \"AGRA\",\n \"usersubtype\": \"hm\",\n \"grade\": [],\n \"usersignintype\": \"Self-Signed-In\",\n \"usertype\": \"administrator\",\n \"userlogintype\": \"administrator\",\n \"state\": \"Uttar Pradesh\"\n },\n \"@timestamp\": \"2022-11-16T09:45:49.682Z\",\n \"devicedata\": {\n \"statecustomcode\": \"29\",\n \"country\": \"India\",\n \"iso3166statecode\": \"IN-KA\",\n \"city\": \"Bengaluru\",\n \"countrycode\": \"IN\",\n \"state\": \"Karnataka\",\n \"devicespec\": {\n \"idisk\": \"106.47\",\n \"webview\": \"107.0.5304.105\",\n \"os\": \"Android 12\",\n \"scrn\": \"6.53\",\n \"sims\": \"-1\",\n \"cpu\": \"abi: arm64-v8a processor\\t: 0 \",\n \"id\": \"ac4ad4ac3feda0f2b17835b81e736c88c194dc89\",\n \"camera\": \"\",\n \"edisk\": \"106.27\",\n \"make\": \"vivo 1915\"\n },\n \"statecode\": \"KA\",\n \"firstaccess\": 1660038763481,\n \"districtcustom\": \"BENGALURU URBAN SOUTH\",\n \"statecustomname\": \"Karnataka\",\n \"userdeclared\": {\n \"district\": \"AGRA\",\n \"state\": \"Uttar Pradesh\"\n }\n },\n \"context\": {\n \"cdata\": [\n {\n \"id\": \"39901401-de90-4c4f-931b-fadf34522e26\",\n \"type\": \"UserSession\"\n }\n ],\n \"env\": \"sdk\",\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"preprod.diksha.app\",\n \"pid\": \"sunbird.app\",\n \"ver\": \"4.10.1023preproduction\"\n },\n \"sid\": \"39901401-de90-4c4f-931b-fadf34522e26\",\n \"did\": \"ac4ad4ac3feda0f2b17835b81e736c88c194dc89\",\n \"rollup\": {\n \"l1\": \"0126796199493140480\"\n }\n },\n \"object\": {\n \"id\": \"\",\n \"type\": \"\",\n \"version\": \"\",\n \"rollup\": {}\n }\n },\n {\n \"eid\": \"SEARCH\",\n \"ver\": \"3.0\",\n \"syncts\": 1668590880098,\n \"ets\": 1668590869525,\n \"flags\": {\n \"pp_duplicate_skipped\": true,\n \"pp_validation_processed\": true,\n \"device_denorm\": true,\n \"loc_denorm\": true\n },\n \"derivedlocationdata\": {\n \"district\": \"BENGALURU URBAN SOUTH\",\n \"from\": \"user-declared\",\n \"state\": \"Karnataka\"\n },\n \"mid\": \"f47ac10c-58cc-4372-a567-0e02b2c3d479\",\n \"type\": \"events\",\n \"actor\": {\n \"id\": \"b6d0a460-430e-4077-8c50-dfe38e9aade3\",\n \"type\": \"Consumer\"\n },\n \"edata\": {\n \"size\": 1,\n \"query\": \"\",\n \"filters\": {\n \"slug\": \"ntp\",\n \"isTenant\": true,\n \"make\": {\n \"type\": \"apple\",\n \"vers\": 1\n }\n },\n \"sort\": {},\n \"type\": \"Org_alias\",\n \"topn\": [\n {\n \"id\": \"01268904781886259221\"\n }\n ]\n },\n \"@timestamp\": \"2022-11-16T09:28:00.098Z\",\n \"devicedata\": {\n \"statecustomcode\": \"\",\n \"country\": \"\",\n \"iso3166statecode\": \"\",\n \"city\": \"\",\n \"countrycode\": \"\",\n \"state\": \"\",\n \"devicespec\": {},\n \"statecode\": \"\",\n \"firstaccess\": 1668580743870,\n \"districtcustom\": \"\",\n \"statecustomname\": \"\",\n \"userdeclared\": {\n \"district\": \"BENGALURU URBAN SOUTH\",\n \"state\": \"Karnataka\"\n }\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"preprod.diksha.portal\",\n \"pid\": \"learner-service\",\n \"ver\": \"4.3.0\"\n },\n \"env\": \"User\",\n \"did\": \"0b83cf8293ff83aff1b20ee9a800f9d4\",\n \"cdata\": [\n {\n \"id\": \"a600ebb2-4acc-6c30-8b08-b804a0d386e1\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n }\n },\n {\n \"eid\": \"SEARCH\",\n \"ver\": \"3.0\",\n \"syncts\": 1668590880098,\n \"ets\": 1668590869525,\n \"flags\": {\n \"pp_duplicate_skipped\": true,\n \"pp_validation_processed\": true,\n \"device_denorm\": true,\n \"loc_denorm\": true\n },\n \"derivedlocationdata\": {\n \"district\": \"BENGALURU URBAN SOUTH\",\n \"from\": \"user-declared\",\n \"state\": \"Karnataka\"\n },\n \"mid\": \"a610ebb2-4acd-6d30-8b08-b804a0d386e1\",\n \"type\": \"events\",\n \"actor\": {\n \"id\": \"3d45a701-2ebc-4111-b4c6-6b5ad9ec6e0c\",\n \"type\": \"Consumer\"\n },\n \"edata\": {\n \"size\": 1,\n \"query\": \"\",\n \"filters\": {\n \"slug\": \"ntp\",\n \"isTenant\": true,\n \"make\": {\n \"type\": \"apple\"\n }\n },\n \"sort\": {},\n \"type\": \"Org_alias\",\n \"topn\": [\n {\n \"id\": \"01268904781886259221\"\n }\n ]\n },\n \"@timestamp\": \"2022-11-16T09:28:00.098Z\",\n \"devicedata\": {\n \"statecustomcode\": \"\",\n \"country\": \"\",\n \"iso3166statecode\": \"\",\n \"city\": \"\",\n \"countrycode\": \"\",\n \"state\": \"\",\n \"devicespec\": {},\n \"statecode\": \"\",\n \"firstaccess\": 1668580743870,\n \"districtcustom\": \"\",\n \"statecustomname\": \"\",\n \"userdeclared\": {\n \"district\": \"BENGALURU URBAN SOUTH\",\n \"state\": \"Karnataka\"\n }\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"preprod.diksha.portal\",\n \"pid\": \"learner-service\",\n \"ver\": \"4.3.0\"\n },\n \"env\": \"User\",\n \"did\": \"0b83cf8293ff83aff1b20ee9a800f9d4\",\n \"cdata\": [\n {\n \"id\": \"a600ebb2-4acc-6c30-8b08-b804a0d386e1\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n }\n }\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "{{HOST_IP}}/obsrv/v1/data/create/sb-telemetry" + }, + "response": [] + }, + { + "name": "Data Ingest individual", + "request": { + "auth": { + "type": "noauth" + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"data\": {\n \"event\": {\n \"eid\": \"INTERACT\",\n \"date\": \"2022-01-01\",\n \"ver\": \"3.0\",\n \"syncts\": 1668591949682,\n \"ets\": 1668591949682,\n \"flags\": {\n \"ex_processed\": true,\n \"pp_validation_processed\": true,\n \"pp_duplicate_skipped\": true,\n \"device_denorm\": true,\n \"user_denorm\": true,\n \"loc_denorm\": true\n },\n \"derivedlocationdata\": {\n \"district\": \"AGRA\",\n \"from\": \"user-profile\",\n \"state\": \"Uttar Pradesh\"\n },\n \"mid\": \"6c3fc8d2-357e-479b-b0c0-afdde6e5c6c0\",\n \"type\": \"events\",\n \"actor\": {\n \"type\": \"User\",\n \"id\": \"b6d0a460-430e-4077-8c50-dfe38e9aade3\"\n },\n \"edata\": {\n \"type\": \"session\"\n },\n \"userdata\": {\n \"subject\": [],\n \"district\": \"AGRA\",\n \"usersubtype\": \"hm\",\n \"grade\": [],\n \"usersignintype\": \"Self-Signed-In\",\n \"usertype\": \"administrator\",\n \"userlogintype\": \"administrator\",\n \"state\": \"Uttar Pradesh\"\n },\n \"@timestamp\": \"2022-11-16T09:45:49.682Z\",\n \"devicedata\": {\n \"statecustomcode\": \"29\",\n \"country\": \"India\",\n \"iso3166statecode\": \"IN-KA\",\n \"city\": \"Bengaluru\",\n \"countrycode\": \"IN\",\n \"state\": \"Karnataka\",\n \"devicespec\": {\n \"idisk\": \"106.47\",\n \"webview\": \"107.0.5304.105\",\n \"os\": \"Android 12\",\n \"scrn\": \"6.53\",\n \"sims\": \"-1\",\n \"cpu\": \"abi: arm64-v8a processor\\t: 0 \",\n \"id\": \"ac4ad4ac3feda0f2b17835b81e736c88c194dc89\",\n \"camera\": \"\",\n \"edisk\": \"106.27\",\n \"make\": \"vivo 1915\"\n },\n \"statecode\": \"KA\",\n \"firstaccess\": 1660038763481,\n \"districtcustom\": \"BENGALURU URBAN SOUTH\",\n \"statecustomname\": \"Karnataka\",\n \"userdeclared\": {\n \"district\": \"AGRA\",\n \"state\": \"Uttar Pradesh\"\n }\n },\n \"context\": {\n \"cdata\": [\n {\n \"id\": \"a3c784f0-61d8-43e4-a92a-373fd4338c1d\",\n \"type\": \"UserSession\"\n }\n ],\n \"env\": \"sdk\",\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"preprod.diksha.app\",\n \"pid\": \"sunbird.app\",\n \"ver\": \"4.10.1023preproduction\"\n },\n \"sid\": \"a3c784f0-61d8-43e4-a92a-373fd4338c1d\",\n \"did\": \"ac4ad4ac3feda0f2b17835b81e736c88c194dc89\",\n \"rollup\": {\n \"l1\": \"0126796199493140480\"\n }\n },\n \"object\": {\n \"id\": \"\",\n \"type\": \"\",\n \"version\": \"\",\n \"rollup\": {}\n }\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "{{HOST_IP}}/obsrv/v1/data/create/sb-telemetry" + }, + "response": [] + }, + { + "name": "Data Ingest Master", + "request": { + "auth": { + "type": "noauth" + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"data\": {\n \"event\": {\n \"subject\": [\n \"Mathematics\"\n ],\n \"channel\": \"Future Assurance Consultant\",\n \"language\": [\n \"English\"\n ],\n \"id\": \"311763b2-d7de-4d46-8803-20407eaa3403\",\n \"firstName\": \"Karan\",\n \"lastName\": \"Panicker\",\n \"mobile\": \"+91-602-8988588\",\n \"email\": \"Karan_Panicker@obsrv.ai\",\n \"state\": \"Gujarat\",\n \"district\": \"Bedfordshire\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "{{HOST_IP}}/obsrv/v1/data/create/sb-telemetry-user" + }, + "response": [] + }, + { + "name": "submit ingestion", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"type\": \"kafka\",\n \"spec\": {\n \"dataSchema\": {\n \"dataSource\": \"sb-telemetry.1_DAY\",\n \"dimensionsSpec\": {\n \"dimensions\": [\n {\n \"type\": \"string\",\n \"name\": \"eid\"\n },\n {\n \"type\": \"string\",\n \"name\": \"ver\"\n },\n {\n \"type\": \"long\",\n \"name\": \"syncts\"\n },\n {\n \"type\": \"boolean\",\n \"name\": \"flags_ex_processed\"\n },\n {\n \"type\": \"boolean\",\n \"name\": \"flags_pp_validation_processed\"\n },\n {\n \"type\": \"boolean\",\n \"name\": \"flags_pp_duplicate_skipped\"\n },\n {\n \"type\": \"boolean\",\n \"name\": \"flags_user_denorm\"\n },\n {\n \"type\": \"boolean\",\n \"name\": \"flags_device_denorm\"\n },\n {\n \"type\": \"boolean\",\n \"name\": \"flags_loc_denorm\"\n },\n {\n \"type\": \"boolean\",\n \"name\": \"flags_content_denorm\"\n },\n {\n \"type\": \"boolean\",\n \"name\": \"flags_coll_denorm\"\n },\n {\n \"type\": \"string\",\n \"name\": \"mid\"\n },\n {\n \"type\": \"string\",\n \"name\": \"actor_id\"\n },\n {\n \"type\": \"string\",\n \"name\": \"actor_type\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_type\"\n },\n {\n \"type\": \"long\",\n \"name\": \"edata_duration\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_query\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_objectType\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_version\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_status\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_id\"\n },\n {\n \"type\": \"boolean\",\n \"name\": \"edata_filters_isRootOrg\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_filters_trackable_enabled\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_identifier\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_contentType\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_mimeType\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_hashTagId\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_filters_createdBy\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_mediaType\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_filters_origin\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_filters_primaryCategory\"\n },\n {\n \"name\": \"edata_filters_trackable\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_sort_lastUpdatedOn\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_topn\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_pageid\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_uri\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_subtype\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_id\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_data\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_uaspec_agent\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_uaspec_ver\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_uaspec_system\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_uaspec_platform\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_uaspec_raw\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_state\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_props\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_prevstate\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_dspec_os\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_dspec_make\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_dspec_id\"\n },\n {\n \"type\": \"long\",\n \"name\": \"edata_dspec_idisk\"\n },\n {\n \"type\": \"long\",\n \"name\": \"edata_dspec_edisk\"\n },\n {\n \"type\": \"long\",\n \"name\": \"edata_dspec_scrn\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_dspec_camera\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_dspec_cpu\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_dspec_webview\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_extra_pos\"\n },\n {\n \"type\": \"array\",\n \"name\": \"edata_extra_values\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_extra_query\"\n },\n {\n \"type\": \"string\",\n \"name\": \"edata_mode\"\n },\n {\n \"type\": \"string\",\n \"name\": \"@timestamp\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_channel\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_pdata_id\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_pdata_ver\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_pdata_pid\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_env\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_sid\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_rollup_l1\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_rollup_l2\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_rollup_l3\"\n },\n {\n \"type\": \"array\",\n \"name\": \"context_cdata\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_did\"\n },\n {\n \"type\": \"string\",\n \"name\": \"context_uid\"\n },\n {\n \"type\": \"string\",\n \"name\": \"object_id\"\n },\n {\n \"type\": \"string\",\n \"name\": \"object_type\"\n },\n {\n \"type\": \"string\",\n \"name\": \"object_ver\"\n },\n {\n \"type\": \"string\",\n \"name\": \"object_rollup_l1\"\n },\n {\n \"type\": \"string\",\n \"name\": \"object_version\"\n },\n {\n \"type\": \"array\",\n \"name\": \"tags\"\n }\n ]\n },\n \"timestampSpec\": {\n \"column\": \"ets\",\n \"format\": \"auto\"\n },\n \"metricsSpec\": [\n {\n \"type\": \"doubleSum\",\n \"name\": \"edata_size\",\n \"fieldName\": \"edata_size\"\n },\n {\n \"type\": \"doubleSum\",\n \"name\": \"edata_filters_compatibilityLevel_min\",\n \"fieldName\": \"edata_filters_compatibilityLevel_min\"\n },\n {\n \"type\": \"doubleSum\",\n \"name\": \"edata_filters_compatibilityLevel_max\",\n \"fieldName\": \"edata_filters_compatibilityLevel_max\"\n },\n {\n \"type\": \"doubleSum\",\n \"name\": \"edata_dspec_sims\",\n \"fieldName\": \"edata_dspec_sims\"\n }\n ],\n \"granularitySpec\": {\n \"type\": \"uniform\",\n \"segmentGranularity\": \"DAY\",\n \"rollup\": false\n }\n },\n \"tuningConfig\": {\n \"type\": \"kafka\",\n \"maxBytesInMemory\": 134217728,\n \"maxRowsPerSegment\": 500000,\n \"logParseExceptions\": true\n },\n \"ioConfig\": {\n \"type\": \"kafka\",\n \"topic\": \"sb-telemetry\",\n \"consumerProperties\": {\n \"bootstrap.servers\": \"kafka-headless.kafka.svc:9092\"\n },\n \"taskCount\": 1,\n \"replicas\": 1,\n \"taskDuration\": \"PT1H\",\n \"useEarliestOffset\": true,\n \"completionTimeout\": \"PT1H\",\n \"inputFormat\": {\n \"type\": \"json\",\n \"flattenSpec\": {\n \"useFieldDiscovery\": true,\n \"fields\": [\n {\n \"type\": \"path\",\n \"expr\": \"$.eid\",\n \"name\": \"eid\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.ver\",\n \"name\": \"ver\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.syncts\",\n \"name\": \"syncts\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.ets\",\n \"name\": \"ets\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.flags.ex_processed\",\n \"name\": \"flags_ex_processed\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.flags.pp_validation_processed\",\n \"name\": \"flags_pp_validation_processed\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.flags.pp_duplicate_skipped\",\n \"name\": \"flags_pp_duplicate_skipped\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.flags.user_denorm\",\n \"name\": \"flags_user_denorm\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.flags.device_denorm\",\n \"name\": \"flags_device_denorm\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.flags.loc_denorm\",\n \"name\": \"flags_loc_denorm\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.flags.content_denorm\",\n \"name\": \"flags_content_denorm\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.flags.coll_denorm\",\n \"name\": \"flags_coll_denorm\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.mid\",\n \"name\": \"mid\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.actor.id\",\n \"name\": \"actor_id\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.actor.type\",\n \"name\": \"actor_type\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.type\",\n \"name\": \"edata_type\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.duration\",\n \"name\": \"edata_duration\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.query\",\n \"name\": \"edata_query\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.objectType[*]\",\n \"name\": \"edata_filters_objectType\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.version[*]\",\n \"name\": \"edata_filters_version\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.status[*]\",\n \"name\": \"edata_filters_status\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.id[*]\",\n \"name\": \"edata_filters_id\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.isRootOrg\",\n \"name\": \"edata_filters_isRootOrg\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.trackable.enabled\",\n \"name\": \"edata_filters_trackable_enabled\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.identifier[*]\",\n \"name\": \"edata_filters_identifier\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.contentType[*]\",\n \"name\": \"edata_filters_contentType\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.mimeType[*]\",\n \"name\": \"edata_filters_mimeType\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.hashTagId[*]\",\n \"name\": \"edata_filters_hashTagId\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.createdBy\",\n \"name\": \"edata_filters_createdBy\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.mediaType[*]\",\n \"name\": \"edata_filters_mediaType\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.origin\",\n \"name\": \"edata_filters_origin\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.primaryCategory[*]\",\n \"name\": \"edata_filters_primaryCategory\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.trackable\",\n \"name\": \"edata_filters_trackable\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.sort.lastUpdatedOn\",\n \"name\": \"edata_sort_lastUpdatedOn\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.topn[*]\",\n \"name\": \"edata_topn\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.pageid\",\n \"name\": \"edata_pageid\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.uri\",\n \"name\": \"edata_uri\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.subtype\",\n \"name\": \"edata_subtype\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.id\",\n \"name\": \"edata_id\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.data\",\n \"name\": \"edata_data\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.uaspec.agent\",\n \"name\": \"edata_uaspec_agent\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.uaspec.ver\",\n \"name\": \"edata_uaspec_ver\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.uaspec.system\",\n \"name\": \"edata_uaspec_system\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.uaspec.platform\",\n \"name\": \"edata_uaspec_platform\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.uaspec.raw\",\n \"name\": \"edata_uaspec_raw\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.state\",\n \"name\": \"edata_state\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.props[*]\",\n \"name\": \"edata_props\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.prevstate\",\n \"name\": \"edata_prevstate\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.os\",\n \"name\": \"edata_dspec_os\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.make\",\n \"name\": \"edata_dspec_make\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.id\",\n \"name\": \"edata_dspec_id\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.idisk\",\n \"name\": \"edata_dspec_idisk\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.edisk\",\n \"name\": \"edata_dspec_edisk\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.scrn\",\n \"name\": \"edata_dspec_scrn\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.camera\",\n \"name\": \"edata_dspec_camera\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.cpu\",\n \"name\": \"edata_dspec_cpu\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.webview\",\n \"name\": \"edata_dspec_webview\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.extra.pos[*]\",\n \"name\": \"edata_extra_pos\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.extra.values[*]\",\n \"name\": \"edata_extra_values\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.extra.query\",\n \"name\": \"edata_extra_query\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.mode\",\n \"name\": \"edata_mode\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.@timestamp\",\n \"name\": \"@timestamp\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.channel\",\n \"name\": \"context_channel\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.pdata.id\",\n \"name\": \"context_pdata_id\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.pdata.ver\",\n \"name\": \"context_pdata_ver\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.pdata.pid\",\n \"name\": \"context_pdata_pid\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.env\",\n \"name\": \"context_env\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.sid\",\n \"name\": \"context_sid\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.rollup.l1\",\n \"name\": \"context_rollup_l1\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.rollup.l2\",\n \"name\": \"context_rollup_l2\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.rollup.l3\",\n \"name\": \"context_rollup_l3\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.cdata[*]\",\n \"name\": \"context_cdata\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.did\",\n \"name\": \"context_did\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.context.uid\",\n \"name\": \"context_uid\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.object.id\",\n \"name\": \"object_id\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.object.type\",\n \"name\": \"object_type\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.object.ver\",\n \"name\": \"object_ver\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.object.rollup.l1\",\n \"name\": \"object_rollup_l1\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.object.version\",\n \"name\": \"object_version\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.tags[*]\",\n \"name\": \"tags\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.size\",\n \"name\": \"edata_size\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.compatibilityLevel.min\",\n \"name\": \"edata_filters_compatibilityLevel_min\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.filters.compatibilityLevel.max\",\n \"name\": \"edata_filters_compatibilityLevel_max\"\n },\n {\n \"type\": \"path\",\n \"expr\": \"$.edata.dspec.sims\",\n \"name\": \"edata_dspec_sims\"\n }\n ]\n }\n },\n \"appendToExisting\": false\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "{{HOST_IP}}/obsrv/v1/data/submit/ingestion" + }, + "response": [] + } + ] + }, + { + "name": "Exhaust APIs", + "item": [ + { + "name": "New Request", + "request": { + "method": "GET", + "header": [ + { + "key": "Cookie", + "value": "connect.sid=s%3At7dvK9OasA2_wqELulCynBGChPeHBR1F.wl6k5o703EwIao8jzQ0STmEiOa2klV4%2FhtkMmbSRHMc; connect.sid=s%3AjiH_l418fQEJFLQh-gRbjntH3DNOJ_Pl.VEl9slC2tfMfeoHjlmwgUJpNzDuKx9AyXkoomCnT6TY" + } + ], + "url": { + "raw": "{{HOST_IP}}/obsrv/v1/data/exhaust/sb-telemetry?from=2023-07-01&to=2023-07-31&type=transformed", + "host": [ + "{{HOST_IP}}" + ], + "path": [ + "obsrv", + "v1", + "data", + "exhaust", + "sb-telemetry" + ], + "query": [ + { + "key": "from", + "value": "2023-07-01" + }, + { + "key": "to", + "value": "2023-07-31" + }, + { + "key": "type", + "value": "transformed" + } + ] + } + }, + "response": [] + } + ] + } + ] +} \ No newline at end of file diff --git a/api-service/redoc-static.html b/api-service/redoc-static.html new file mode 100644 index 00000000..034bd56d --- /dev/null +++ b/api-service/redoc-static.html @@ -0,0 +1,538 @@ + + + + + + Obsrv APIS + + + + + + + + + +
API docs by Redocly

Obsrv APIS (1.0.0)

Download OpenAPI specification:Download

Sunbird Obsrv: example@gmail.com URL: # License: All rights reserved

Obsrv is a set of APIs that provide access to a variety of data sources and datasets. These APIs can be used to query and analyze different types of events, as well as to manage data sources and datasets.

+

This OpenAPI document provides a detailed specification for the Obsrv APIs,including information about the endpoints, parameters, responses, and authentication requirements.

+

Obsrv Query APIs

The Query API provides the the ability to query various type of events from the underlying data sources.

+

Perform a native query to fetch data from analytical data source

This endpoint of the READ API provides a way to construction of the queries to read data such as a Native +JSON query. The READ API also allows filters on the date ranges within which the query will be applied to. +The context information in the READ API is required.

+
Request Body schema: application/json
object
query
object

Responses

Request samples

Content type
application/json
{
  • "context": {
    },
  • "query": {
    }
}

Response samples

Content type
application/json
{
  • "id": "obsrv.native.query",
  • "ver": "v1",
  • "ts": 1672679516875,
  • "params": {
    },
  • "responseCode": "OK",
  • "result": [
    ]
}

Perform a sql query to fetch data from analytical data source

This endpoint of the READ API provides a way to construction of the queries to read data such as a Native +JSON query. The READ API also allows filters on the date ranges within which the query will be applied to. +The context information in the READ API is optional.

+
Request Body schema: application/json
object
object

Responses

Request samples

Content type
application/json
{
  • "context": {
    },
  • "querySql": {
    }
}

Response samples

Content type
application/json
{
  • "id": "obsrv.sql.query",
  • "ver": "v1",
  • "ts": 1672679516875,
  • "params": {
    },
  • "responseCode": "OK",
  • "result": [
    ]
}

Obsrv Data IN API

Ingests data from the request body into a Kafka topic.

+

The API designed for ingesting data into Kafka.

"When processing batch events, the API expects the key to be labeled as 'events.' However, when dealing with individual events, the key should be identified as 'event.'"

+
path Parameters
datasetId
required
string
Request Body schema: application/json
data
object

Responses

Request samples

Content type
application/json
Example
{
  • "data": {
    }
}

Response samples

Content type
application/json
{
  • "id": "obsrv.dataset.data.in",
  • "ver": "v1",
  • "ts": 1679916559382,
  • "params": {
    },
  • "responseCode": "OK",
  • "result": {
    }
}

Obsrv Dataset APIs

The Dataset APIs provide a way to interact with a database and allow you to create, read, update records in the database, as well as list multiple records based on certain criteria.

+

creating datasets into postgres tables

This API allows you to create new datasets in the PostgreSQL database used by the analytical data source.

+
Request Body schema: application/json
id
string
dataset_id
string
name
string
type
string
Enum: "dataset" "master-dataset"
object
object
object
object
object
object
tags
Array of strings
status
string
Enum: "Live" "Retired"
created_by
string
updated_by
string
published_date
string

Responses

Request samples

Content type
application/json
{
  • "dataset_id": "obsrv-telemetry-events",
  • "name": "obsrv-telemetry-events",
  • "type": "dataset",
  • "router_config": {
    },
  • "tags": [ ],
  • "data_schema": {
    },
  • "status": "Live",
  • "published_date": "2023-03-24 12:19:32.091544"
}

Response samples

Content type
application/json
{
  • "id": "obsrv.config.dataset.save",
  • "ver": "v1",
  • "ts": 1679916559382,
  • "params": {
    },
  • "responseCode": "OK",
  • "result": {
    }
}

API to update existing records in database

This API allows you to update existing datasets in the PostgreSQL database used by the analytical data source.

+
Request Body schema: application/json
id
string
dataset_id
string
name
string
type
string
Enum: "dataset" "master-dataset"
object
object
object
object
object
object
tags
Array of strings
status
string
Enum: "Live" "Retired"
created_by
string
updated_by
string
published_date
string

Responses

Request samples

Content type
application/json
{
  • "dataset_id": "obsrv-telemetry-events",
  • "type": "dataset",
  • "name": "obsrv-telemetry-events",
  • "router_config": {
    },
  • "denorm_config": {
    },
  • "dataset_config": {
    },
  • "extraction_config": {
    }
}

Response samples

Content type
application/json
{
  • "id": "obsrv.config.dataset.update",
  • "ver": "v1",
  • "ts": 1679916559382,
  • "params": {
    },
  • "responseCode": "OK",
  • "result": {
    }
}

API to read records from database

This API allows you to retrieve records from a dataset in the PostgreSQL. A successful response will return a status code of 200. If the dataset does not exist, a status code of 404 will be returned.

+
path Parameters
datasetId
required
string <uuid>

Responses

Response samples

Content type
application/json
{
  • "id": "obsrv.config.dataset.read",
  • "ver": "v1",
  • "ts": 1672679516875,
  • "params": {
    },
  • "responseCode": "OK",
  • "result": {
    }
}

API to list records from database

This API allows you to retrieve a list of records from a dataset in the PostgreSQL database used by the analytical data source. You can specify filters to narrow down the results.

+
Request Body schema: application/json
filters
object

Responses

Request samples

Content type
application/json
{
  • "filters": {
    }
}

Response samples

Content type
application/json
{
  • "id": "obsrv.config.dataset.list",
  • "ver": "v1",
  • "ts": 1672679516875,
  • "params": {
    },
  • "responseCode": "OK",
  • "result": [
    ]
}

Obsrv Datasource APIs

The Datsources APIs provide a way to interact with a database and allow you to create, read, update records in the database, as well as list multiple records based on certain criteria.

+

creating datasources into postgres tables

This API allows you to create new datasource in the PostgreSQL database. While saving a datasource, it is required that the datasource field in ingestion spec must match with datasource_ref.

+
Request Body schema: application/json
id
string
dataset_id
string
ingestion_spec
object
datasource
string
datasource_ref
string
retention_period
object
archival_policy
object
purge_policy
object
backup_config
object
status
string
Enum: "Live" "Retired"
created_by
string
updated_by
string
published_date
string
metadata
object

Responses

Request samples

Content type
application/json
{
  • "dataset_id": "obsrv-telemetry-events",
  • "ingestion_spec": {
    },
  • "datasource": "obsrv-telemetry-events",
  • "datasource_ref": "obsrv-telemetry-events",
  • "published_date": "2023-03-27T11:40:43.269Z",
  • "status": "Live",
  • "metadata": {
    }
}

Response samples

Content type
application/json
{
  • "id": "obsrv.config.datasource.create",
  • "ver": "v1",
  • "ts": 1679916559382,
  • "params": {
    },
  • "responseCode": "OK",
  • "result": {
    }
}

API to update existing records in database

This API allows you to update existing datasource in the PostgreSQL database. While saving a datasource, it is required that the datasource field in ingestion spec must match with datasource_ref.

+
Request Body schema: application/json
id
string
dataset_id
string
ingestion_spec
object
datasource
string
datasource_ref
string
retention_period
object
archival_policy
object
purge_policy
object
backup_config
object
status
string
Enum: "Live" "Retired"
created_by
string
updated_by
string
published_date
string
metadata
object

Responses

Request samples

Content type
application/json
{
  • "dataset_id": "obsrv-telemetry-events",
  • "datasource": "obsrv-telemetry-events",
  • "backup_config": {
    }
}

Response samples

Content type
application/json
{
  • "id": "obsrv.config.datasource.update",
  • "ver": "v1",
  • "ts": 1679916559382,
  • "params": {
    },
  • "responseCode": "OK",
  • "result": {
    }
}

API to read records from database

This API allows you to retrieve records from a datasource in the PostgreSQL. A successful response will return a status code of 200. If the datasource does not exist, a status code of 404 will be returned

+
path Parameters
datasourceId
required
string <uuid>

Responses

Response samples

Content type
application/json
{
  • "id": "obsrv.config.datasource.read",
  • "ver": "v1",
  • "ts": 1672679516875,
  • "params": {
    },
  • "responseCode": "OK",
  • "result": {
    }
}

API to list records from database

This API allows you to retrieve a list of records from a dataset in the PostgreSQL database used by the analytical data source. You can specify filters to narrow down the results.

+
Request Body schema: application/json
filters
object

Responses

Request samples

Content type
application/json
{
  • "filters": {
    }
}

Response samples

Content type
application/json
{
  • "id": "obsrv.config.datasource.list",
  • "ver": "v1",
  • "ts": 1672679516875,
  • "params": {
    },
  • "responseCode": "OK",
  • "result": [
    ]
}

API to submit ingestion spec to analytical database

Request Body schema: application/json
object (SubmitIngestion)

Responses

Request samples

Content type
application/json
{
  • "type": "kafka",
  • "spec": {
    }
}

Response samples

Content type
application/json
{
  • "id": "obsrv.submit.ingestion",
  • "ver": "v1",
  • "ts": 1693891173180,
  • "params": {
    },
  • "responseCode": "OK",
  • "result": {
    }
}

Obsrv Dataset Source Config APIs

The Dataset Source Config APIs provide a way to interact with a database and allow you to create, read, update records in the database, as well as list multiple records based on certain criteria.

+

creating datasets into postgres tables

This API allows you to create new datasets in the PostgreSQL database used by the analytical data source.

+
Request Body schema: application/json
id
string
dataset_id
string
connector_type
string
connector_config
object
status
string
connector_stats
object
created_by
string
updated_by
string
published_date
string

Responses

Request samples

Content type
application/json
{
  • "dataset_id": "obsrv-telemetry-events",
  • "connector_type": "kafka",
  • "connector_config": {
    },
  • "status": "Live",
  • "published_date": "2023-03-24 12:19:32.091544"
}

Response samples

Content type
application/json
{
  • "id": "obsrv.config.dataset.source.config.create",
  • "ver": "v1",
  • "ts": 1679916559382,
  • "params": {
    },
  • "responseCode": "OK",
  • "result": {
    }
}

API to update existing records in database

This API allows you to update existing datasets in the PostgreSQL database used by the analytical data source.

+
Request Body schema: application/json
id
string
dataset_id
string
connector_type
string
connector_config
object
status
string
connector_stats
object
created_by
string
updated_by
string
published_date
string

Responses

Request samples

Content type
application/json
{
  • "dataset_id": "obsrv-telemetry-events",
  • "connector_type": "kafka",
  • "connector_config": {
    },
  • "status": "Live"
}

Response samples

Content type
application/json
{
  • "id": "obsrv.config.dataset.source.config.update",
  • "ver": "v1",
  • "ts": 1679916559382,
  • "params": {
    },
  • "responseCode": "OK",
  • "result": {
    }
}

API to read records from database

This API allows you to retrieve records from a dataset in the PostgreSQL. A successful response will return a status code of 200. If the dataset does not exist, a status code of 404 will be returned.

+
path Parameters
datasetId
required
string

Responses

Response samples

Content type
application/json
{
  • "id": "obsrv.config.dataset.source.config.read",
  • "ver": "v1",
  • "ts": 1672679516875,
  • "params": {
    },
  • "responseCode": "OK",
  • "result": {
    }
}

API to list records from database

This API allows you to retrieve a list of records from a dataset in the PostgreSQL database used by the analytical data source. You can specify filters to narrow down the results.

+
Request Body schema: application/json
filters
object

Responses

Request samples

Content type
application/json
{
  • "filters": {
    }
}

Response samples

Content type
application/json
{
  • "id": "obsrv.config.dataset.source.config.list",
  • "ver": "v1",
  • "ts": 1672679516875,
  • "params": {
    },
  • "responseCode": "OK",
  • "result": [
    ]
}

Obsrv Data Exhaust APIs

Data Exhaust APIs are used for accessing Backup files from Cloud providers

+

API to access backup files from Cloud providers

path Parameters
datasetId
required
string
query Parameters
required
object or object (DataExhaust)
Example: from=2023-06-01&to=2023-06-30&type=transformed

Responses

Response samples

Content type
application/json
{
  • "id": "obsrv.dataset.data.exhaust",
  • "ver": "v1",
  • "ts": 1693891173180,
  • "params": {
    },
  • "responseCode": "OK",
  • "result": {
    }
}
+ + + + diff --git a/api-service/src/configs/RoutesConfig.ts b/api-service/src/configs/RoutesConfig.ts index 42ebbeca..60e1e27b 100644 --- a/api-service/src/configs/RoutesConfig.ts +++ b/api-service/src/configs/RoutesConfig.ts @@ -116,6 +116,12 @@ export const routesConfig = { path: "/obsrv/v1/data/create/:datasetId", validation_schema: "DataIngestionReq.json", }, + tenant_ingest: { + api_id: "dataset.data.in", + method: "post", + path: "/data/tenant/in/:datasetId", + validation_schema: "DataIngestionReq.json", + }, exhaust: { api_id: "obsrv.dataset.data.exhaust", method: "get", diff --git a/api-service/src/helpers/Datasets.ts b/api-service/src/helpers/Datasets.ts index 672e8055..94a12771 100644 --- a/api-service/src/helpers/Datasets.ts +++ b/api-service/src/helpers/Datasets.ts @@ -4,6 +4,8 @@ import { defaultConfig } from '../resources/schemas/DatasetConfigDefault' import { SchemaMerger } from '../generators/SchemaMerger' import { config } from '../configs/Config' import { DatasetStatus } from '../models/DatasetModels' +import constants from "../resources/Constants.json"; + let schemaMerger = new SchemaMerger() export class Datasets { private id: string @@ -47,10 +49,12 @@ export class Datasets { } public getValues() { + this.validateDenormConfig(); return Object.assign(this.removeNullValues({ id: this.id, dataset_id: this.dataset_id, type: this.type, name: this.name, validation_config: this.validation_config, extraction_config: this.extraction_config, dedup_config: this.dedup_config, data_schema: this.data_schema, router_config: this.router_config, denorm_config: this.denorm_config, dataset_config: this.dataset_config, tags: this.tags, status: this.status, created_by: this.created_by, updated_by: this.updated_by, published_date: this.published_date }), { "updated_date": new Date }) } public setValues() { + this.validateDenormConfig(); return schemaMerger.mergeSchema(this.getDefaults(), this.getValues()) } @@ -69,4 +73,21 @@ export class Datasets { return {...defaultConfig.dataset} } } + + private validateDenormConfig() { + if (this.denorm_config && _.has(this.denorm_config, 'denorm_fields')) { + let duplicatesExist = false; + let denormFields: any = _.get(this.denorm_config, 'denorm_fields', []); + denormFields = _.map(denormFields, (denormField: Record) => _.get(denormField, 'denorm_out_field')); + denormFields.map( + (denormField: string | number) => { + if(_.indexOf(denormFields, denormField) !== _.lastIndexOf(denormFields, denormField)) + duplicatesExist = true; + } + ); + if(duplicatesExist) { + throw constants.DUPLICATE_DENORM_FIELD; + } + } + } } diff --git a/api-service/src/resources/Constants.json b/api-service/src/resources/Constants.json index dd3409cc..44a1f570 100644 --- a/api-service/src/resources/Constants.json +++ b/api-service/src/resources/Constants.json @@ -96,6 +96,11 @@ "status": 404, "code": "NOT_FOUND" }, + "DUPLICATE_DENORM_FIELD": { + "message": "Duplicate found for denorm output key", + "status": 400, + "code": "BAD_REQUEST" + }, "INGESTION_SUBMITTED": "ingestion spec has been submitted successfully", "INGESTION_FAILED_ON_SAVE": "Failed to submit Ingestion Spec, record is not saved" } diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index 978ba6ea..fb964565 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -39,6 +39,7 @@ router.post([`${routesConfig.query.sql_query.path}`, `${routesConfig.query.sql_q /** Ingestor API */ router.post(`${routesConfig.data_ingest.path}`, ResponseHandler.setApiId(routesConfig.data_ingest.api_id), telemetryAuditStart({ action: telemetryActions.ingestEvents, operationType: OperationType.CREATE }), onRequest({ entity: promEntities.data_in }), validationService.validateRequestBody, ingestorService.create); +router.post(`${routesConfig.tenant_ingest.path}`, ResponseHandler.setApiId(routesConfig.tenant_ingest.api_id), telemetryAuditStart({ action: telemetryActions.ingestEvents, operationType: OperationType.CREATE }), onRequest({ entity: promEntities.data_in }), validationService.validateRequestBody, ingestorService.tenant); /** Dataset APIs */ router.post(`${routesConfig.config.dataset.save.path}`, ResponseHandler.setApiId(routesConfig.config.dataset.save.api_id), telemetryAuditStart({ action: telemetryActions.createDataset, operationType: OperationType.CREATE }), validationService.validateRequestBody, datasetService.save); diff --git a/api-service/src/services/IngestorService.ts b/api-service/src/services/IngestorService.ts index 6eb912dc..482cd764 100644 --- a/api-service/src/services/IngestorService.ts +++ b/api-service/src/services/IngestorService.ts @@ -35,6 +35,20 @@ export class IngestorService { ResponseHandler.successResponse(req, res, { status: 200, data: { message: constants.DATASET.CREATED } }); } catch (error: any) { this.errorHandler.handleError(req, res, next, error, false) } } + + public tenant = async (req: Request, res: Response, next: NextFunction) => { + try { + let datasetId = this.getDatasetId(req); + const tenantId = _.get(req.headers, 'x-tenant-id', "default"); + datasetId = `${tenantId}-${datasetId}`; + const validData = await this.validateData(req.body.data, datasetId); + req.body = { ...req.body.data, dataset: datasetId }; + const topic = await this.getTopic(datasetId); + await this.kafkaConnector.execute(req, res, topic); + ResponseHandler.successResponse(req, res, { status: 200, data: { message: constants.DATASET.CREATED } }); + } catch (error: any) { this.errorHandler.handleError(req, res, next, error, false) } + } + public submitIngestion = async (req: Request, res: Response, next: NextFunction) => { try { await wrapperService.submitIngestion(req.body) diff --git a/api-service/src/services/telemetry.ts b/api-service/src/services/telemetry.ts index afe85bb9..e63d1d09 100644 --- a/api-service/src/services/telemetry.ts +++ b/api-service/src/services/telemetry.ts @@ -12,7 +12,7 @@ export enum OperationType { CREATE = 1, UPDATE, PUBLISH, RETIRE, LIST, GET } const kafka = new Kafka({ clientId: telemetryTopic, brokers: brokerServers }); const telemetryEventsProducer = kafka.producer(); -telemetryEventsProducer.connect(); +telemetryEventsProducer.connect().catch(err => console.error("Unable to connect to kafka", err.message)); const getDefaults = () => { return { diff --git a/api-service/src/test/DatasetTestService.spec.ts b/api-service/src/test/DatasetTestService.spec.ts index 8ce71a40..8887163a 100644 --- a/api-service/src/test/DatasetTestService.spec.ts +++ b/api-service/src/test/DatasetTestService.spec.ts @@ -108,7 +108,7 @@ describe("Dataset create API", () => { res.body.params.status.should.be.eq(constants.STATUS.FAILURE) done(); }); - }) + }); it("should not insert record when given invalid schema", (done) => { chai .request(app) @@ -123,7 +123,45 @@ describe("Dataset create API", () => { res.body.params.status.should.be.eq(constants.STATUS.FAILURE) done(); }); - }) + }); + it("should not insert the record when there's a duplicate denorm out field", (done) => { + chai.spy.on(dbConnector, "execute", () => { + return Promise.resolve([]) + }) + chai + .request(app) + .post(config.apiDatasetSaveEndPoint) + .send(TestDataset.DUPLICATE_DENORM_OUT_FIELD) + .end((err, res) => { + res.should.have.status(httpStatus.BAD_REQUEST); + res.body.should.be.a("object") + res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); + res.body.should.have.property("result"); + res.body.id.should.be.eq(routesConfig.config.dataset.save.api_id); + res.body.params.status.should.be.eq(constants.STATUS.FAILURE); + chai.spy.restore(dbConnector, "execute"); + done(); + }); + }); + it("should insert the record when there's no duplicate denorm out field", (done) => { + chai.spy.on(dbConnector, "execute", () => { + return Promise.resolve([]) + }) + chai + .request(app) + .post(config.apiDatasetSaveEndPoint) + .send(TestDataset.VALID_DENORM_OUT_FIELD) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); + res.body.should.have.property("result"); + res.body.id.should.be.eq(routesConfig.config.dataset.save.api_id); + res.body.params.status.should.be.eq(constants.STATUS.SUCCESS); + chai.spy.restore(dbConnector, "execute"); + done(); + }); + }); }) describe("Dataset update API", () => { beforeEach(() => { diff --git a/api-service/src/test/Fixtures.ts b/api-service/src/test/Fixtures.ts index 37d97348..a69fd39c 100644 --- a/api-service/src/test/Fixtures.ts +++ b/api-service/src/test/Fixtures.ts @@ -51,7 +51,9 @@ class TestDataset { public static VALID_LIST_REQUEST_ACTIVE_STATUS = { "filters": { "status": [ DatasetStatus.Live ] } }; public static VALID_LIST_REQUEST_DISABLED_STATUS = { "filters": { "status": [ DatasetStatus.Retired ] } }; public static MISSING_REQUIRED_FIELDS_UPDATE = { "name": "telemetry-raw", "data_schema": { "type": "object", "properties": { "eid": { "type": "string" }, "ver": { "type": "string" }, "syncts": { "type": "integer" }, "ets": { "type": "integer" }, "mid": { "type": "string" }, "actor": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] }, "edata": { "type": "object", "properties": { "type": { "type": "string" } }, "required": [ "type" ] }, "@timestamp": { "type": "string" }, "context": { "type": "object", "properties": { "pdata": { "type": "object", "properties": { "ver": { "type": "string" }, "id": { "type": "string" }, "pid": { "type": "string" } }, "required": [ "ver", "id", "pid" ] }, "did": { "type": "string" }, "env": { "type": "string" }, "channel": { "type": "string" } }, "required": [ "pdata", "did", "env", "channel" ] }, "@version": { "type": "string" }, "object": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] } }, "required": [ "eid", "ver", "syncts", "ets", "mid", "actor", "edata", "@timestamp", "context", "@version", "object" ] }, "router_config": { "topic": "router.topic" }, "status": DatasetStatus.Live, "published_date": "2023-03-14T04:46:33.459Z" }; - public static VALID_RECORD = { "type": "master-dataset", "dataset_id": "3f8b2ba7-9c74-4d7f-8b38-2b0d460b999c", "id": "observations", "name": "telemetry-raw", " validation_config": { "validate": true, "mode": "Strict" }, "extraction_config": { "is_batch_event": false, "extraction_key": "", "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 3 } }, "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 3 }, "data_schema": { "type": "object", "properties": { "eid": { "type": "string" }, "ver": { "type": "string" }, "syncts": { "type": "integer" }, "ets": { "type": "integer" }, "mid": { "type": "string" }, "actor": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] }, "edata": { "type": "object", "properties": { "type": { "type": "string" } }, "required": [ "type" ] }, "@timestamp": { "type": "string" }, "context": { "type": "object", "properties": { "pdata": { "type": "object", "properties": { "ver": { "type": "string" }, "id": { "type": "string" }, "pid": { "type": "string" } }, "required": [ "ver", "id", "pid" ] }, "did": { "type": "string" }, "env": { "type": "string" }, "channel": { "type": "string" } }, "required": [ "pdata", "did", "env", "channel" ] }, "@version": { "type": "string" }, "object": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] } }, "required": [ "eid", "ver", "syncts", "ets", "mid", "actor", "edata", "@timestamp", "context", "@version", "object" ] }, "denorm_config": { "redis_db_host": "redis_host", "redis_db_port": "redis_port", "denorm_fields": { "denorm_key": "", "redis_db": 1, "denorm_out_field": "metadata" } }, "tags": [], "router_config": { "topic": "router.topic" }, "client_state": {}, "status": DatasetStatus.Live, "created_by": "SYSTEM", "updated_by": "SYSTEM", "created_date": "2023-03-13T07:46:06.410Z", "updated_date": "2023-03-14T04:46:33.459Z", "published_date": "2023-03-14T04:46:33.459Z" } + public static VALID_RECORD = { "type": "master-dataset", "dataset_id": "3f8b2ba7-9c74-4d7f-8b38-2b0d460b999c", "id": "observations", "name": "telemetry-raw", " validation_config": { "validate": true, "mode": "Strict" }, "extraction_config": { "is_batch_event": false, "extraction_key": "", "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 3 } }, "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 3 }, "data_schema": { "type": "object", "properties": { "eid": { "type": "string" }, "ver": { "type": "string" }, "syncts": { "type": "integer" }, "ets": { "type": "integer" }, "mid": { "type": "string" }, "actor": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] }, "edata": { "type": "object", "properties": { "type": { "type": "string" } }, "required": [ "type" ] }, "@timestamp": { "type": "string" }, "context": { "type": "object", "properties": { "pdata": { "type": "object", "properties": { "ver": { "type": "string" }, "id": { "type": "string" }, "pid": { "type": "string" } }, "required": [ "ver", "id", "pid" ] }, "did": { "type": "string" }, "env": { "type": "string" }, "channel": { "type": "string" } }, "required": [ "pdata", "did", "env", "channel" ] }, "@version": { "type": "string" }, "object": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] } }, "required": [ "eid", "ver", "syncts", "ets", "mid", "actor", "edata", "@timestamp", "context", "@version", "object" ] }, "denorm_config": { "redis_db_host": "redis_host", "redis_db_port": "redis_port", "denorm_fields": [{ "denorm_key": "", "redis_db": 1, "denorm_out_field": "metadata" }]}, "tags": [], "router_config": { "topic": "router.topic" }, "client_state": {}, "status": DatasetStatus.Live, "created_by": "SYSTEM", "updated_by": "SYSTEM", "created_date": "2023-03-13T07:46:06.410Z", "updated_date": "2023-03-14T04:46:33.459Z", "published_date": "2023-03-14T04:46:33.459Z" }; + public static DUPLICATE_DENORM_OUT_FIELD = { "type": "master-dataset", "dataset_id": "3f8b2ba7-9c74-4d7f-8b38-2b0d460b999c", "id": "observations", "name": "telemetry-raw", " validation_config": { "validate": true, "mode": "Strict" }, "extraction_config": { "is_batch_event": false, "extraction_key": "", "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 3 } }, "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 3 }, "data_schema": { "type": "object", "properties": { "eid": { "type": "string" }, "ver": { "type": "string" }, "syncts": { "type": "integer" }, "ets": { "type": "integer" }, "mid": { "type": "string" }, "actor": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] }, "edata": { "type": "object", "properties": { "type": { "type": "string" } }, "required": [ "type" ] }, "@timestamp": { "type": "string" }, "context": { "type": "object", "properties": { "pdata": { "type": "object", "properties": { "ver": { "type": "string" }, "id": { "type": "string" }, "pid": { "type": "string" } }, "required": [ "ver", "id", "pid" ] }, "did": { "type": "string" }, "env": { "type": "string" }, "channel": { "type": "string" } }, "required": [ "pdata", "did", "env", "channel" ] }, "@version": { "type": "string" }, "object": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] } }, "required": [ "eid", "ver", "syncts", "ets", "mid", "actor", "edata", "@timestamp", "context", "@version", "object" ] }, "denorm_config": { "redis_db_host": "redis_host", "redis_db_port": "redis_port", "denorm_fields": [{ "denorm_key": "test", "redis_db": 1, "denorm_out_field": "metadata" }, { "denorm_key": "test", "redis_db": 1, "denorm_out_field": "metadata" }]}, "tags": [], "router_config": { "topic": "router.topic" }, "client_state": {}, "status": DatasetStatus.Live, "created_by": "SYSTEM", "updated_by": "SYSTEM", "created_date": "2023-03-13T07:46:06.410Z", "updated_date": "2023-03-14T04:46:33.459Z", "published_date": "2023-03-14T04:46:33.459Z" }; + public static VALID_DENORM_OUT_FIELD = { "type": "master-dataset", "dataset_id": "3f8b2ba7-9c74-4d7f-8b38-2b0d460b91ac", "id": "observctions", "name": "telemetay-raw", " validation_config": { "validate": true, "mode": "Strict" }, "extraction_config": { "is_batch_event": false, "extraction_key": "", "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 3 } }, "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 3 }, "data_schema": { "type": "object", "properties": { "eid": { "type": "string" }, "ver": { "type": "string" }, "syncts": { "type": "integer" }, "ets": { "type": "integer" }, "mid": { "type": "string" }, "actor": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] }, "edata": { "type": "object", "properties": { "type": { "type": "string" } }, "required": [ "type" ] }, "@timestamp": { "type": "string" }, "context": { "type": "object", "properties": { "pdata": { "type": "object", "properties": { "ver": { "type": "string" }, "id": { "type": "string" }, "pid": { "type": "string" } }, "required": [ "ver", "id", "pid" ] }, "did": { "type": "string" }, "env": { "type": "string" }, "channel": { "type": "string" } }, "required": [ "pdata", "did", "env", "channel" ] }, "@version": { "type": "string" }, "object": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] } }, "required": [ "eid", "ver", "syncts", "ets", "mid", "actor", "edata", "@timestamp", "context", "@version", "object" ] }, "denorm_config": { "redis_db_host": "redis_host", "redis_db_port": "redis_port", "denorm_fields": [{ "denorm_key": "test", "redis_db": 1, "denorm_out_field": "metadata2" }, { "denorm_key": "test", "redis_db": 1, "denorm_out_field": "metadata" }]}, "tags": [], "router_config": { "topic": "router.topic" }, "client_state": {}, "status": DatasetStatus.Live, "created_by": "SYSTEM", "updated_by": "SYSTEM", "created_date": "2023-03-13T07:46:06.410Z", "updated_date": "2023-03-14T04:46:33.459Z", "published_date": "2023-03-14T04:46:33.459Z" }; } class TestDataSource { diff --git a/api-service/src/validators/RequestsValidator.ts b/api-service/src/validators/RequestsValidator.ts index 3b1c7b8c..15f20e8a 100644 --- a/api-service/src/validators/RequestsValidator.ts +++ b/api-service/src/validators/RequestsValidator.ts @@ -53,6 +53,7 @@ export class RequestsValidator implements IValidator { routesConfig.query.native_query, routesConfig.query.sql_query, routesConfig.data_ingest, + routesConfig.tenant_ingest, routesConfig.config.dataset.save, routesConfig.config.datasource.save, routesConfig.config.dataset.list, diff --git a/command-service/Dockerfile b/command-service/Dockerfile index be98c683..dff12a46 100644 --- a/command-service/Dockerfile +++ b/command-service/Dockerfile @@ -3,11 +3,11 @@ COPY --from=ubuntu /usr/local/bin /usr/local/bin RUN apk update && apk add curl jq && curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && chmod +x kubectl && mv kubectl /usr/local/bin/ -RUN apk add libcrypto3=3.1.4-r0 +RUN apk add libcrypto3=3.1.4-r3 RUN apk upgrade WORKDIR /app -COPY command-service/requirements.txt . +COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt -COPY command-service/src ./src +COPY src ./src WORKDIR /app/src CMD [ "uvicorn", "routes:app", "--host", "0.0.0.0" ] \ No newline at end of file