improve system

ChangeLog.md

@@ -31,7 +31,7 @@
 * How to setup LEO CDP
 * How to monitor system
 
-## [0.9.0] - 2024-09-27
+## [0.9.0] - 2024-10-02
 
 ### Added
 
@@ -90,7 +90,8 @@
 
 ### Fixed
 
-- [Core CDP] fix exporting data in a segment
+- [Segment] improve segment indexing and fix bugs
+- [Core CDP] fix exporting data in a segment 
 - [Core CDP] improve performance of CDP Admin
 - [Profile] fix bugs when update a new profile
 - [Core CDP] segment builder using operator after, before
@@ -152,6 +153,7 @@
 
 ### Changed
 
+- [Admin UI] update font-family: Noto Sans for admin CDP
 - [Segment] improve performance, add new way to refresh data of segment
 - [Event API] add params: sourceip and useragent to save from WooCommerce Data Source
 - [Profile UI] improve touchpoint flow

NOTES-FOR-NEW-SETUP.md

@@ -24,11 +24,12 @@ Command to edit hosts:
 - [the network IP of CDP Admin] cdpsys.admin cdpsys.redis
 - [the network IP of CDP Observer] cdpsys.observer
 
-### Example DNS for 1 CDP Database, 1 CDP Admin and 1 CDP Observer
+### Example DNS for 1 CDP Database, 1 CDP Admin, 1 CDP Observer, 1 local Redis server
 
-	127.0.0.1 cdpsys.database
-	127.0.0.1 cdpsys.admin cdpsys.redis
-	127.0.0.1 cdpsys.observer
+	192.168.1.6 cdpsys.database
+	192.168.1.6 cdpsys.admin             
+	192.168.1.5 cdpsys.observer
+	127.0.0.1 cdpsys.redis
 
 ### Nginx config for db.example.com
 
@@ -98,7 +99,8 @@ Set Linux configs to scale on high load
 	curl -OL https://download.arangodb.com/arangodb311/DEBIAN/Release.key
 	sudo apt-key add - < Release.key
 	echo 'deb https://download.arangodb.com/arangodb311/DEBIAN/ /' | sudo tee /etc/apt/sources.list.d/arangodb.list
-	sudo apt-get install apt-transport-https; sudo apt-get update && sudo apt-get install arangodb3=3.11.9-1
+	sudo apt-get install apt-transport-https; sudo apt-get update; sudo apt-get install arangodb3=3.11.11-1
+	sudo ufw allow from [IP_Observer] to any port 8600
 
 [ArangoDB on CentOS or Rocky Linux](https://idroot.us/install-arangodb-centos-8/)
 
@@ -109,7 +111,7 @@ Set Linux configs to scale on high load
 	>  /var/log/arangodb3/arangod.log
 	nano /etc/arangodb3/arangod.conf
 	systemctl restart arangodb3.service
-	sudo firewall-cmd --add-port=8601/tcp --permanent
+	sudo firewall-cmd --add-port=8600/tcp --permanent
 	sudo firewall-cmd --reload
 
 ## This requirements for HTTP Observer, Admin and Database 

configs/database-query-template.aql

@@ -982,6 +982,29 @@ AQL_GET_PROFILE_BY_ID_FOR_SYSTEM =>
 	RETURN p
 ;
 
+AQL_GET_ACTIVE_PROFILE_IDENTITY_BY_ID =>
+	FOR profile in cdp_profile 
+	FILTER profile._key == @id AND profile.status > 0 
+	RETURN {
+	  id: profile._key,
+	  visitorId: profile.visitorId,
+	  crmRefId: profile.crmRefId,
+	  primaryEmail: profile.primaryEmail,
+	  primaryPhone: profile.primaryPhone,
+	  personaUri: profile.personaUri,
+	  primaryUsername: profile.primaryUsername,
+	  firstName: profile.firstName,
+	  lastName: profile.lastName,
+	  fingerprintId: profile.fingerprintId,
+	  socialMediaProfiles: profile.socialMediaProfiles,
+	  applicationIDs: profile.applicationIDs,
+	  governmentIssuedIDs: profile.governmentIssuedIDs,
+	  loyaltyIDs: profile.loyaltyIDs,
+	  fintechSystemIDs: profile.fintechSystemIDs,
+	  inSegments: profile.inSegments
+	}
+;
+
 AQL_GET_PROFILE_BY_ID_FOR_ADMIN =>
 	FOR p in cdp_profile 
 	FILTER p._key == @id AND (@loginUsername IN p.authorizedViewers[*] OR p.visitorId == @profileVisitorId)
@@ -1216,48 +1239,67 @@ AQL_PROFILE_TIMESERIES_COLLECTOR =>
 ;
 
 AQL_INSERT_SEGMENT_REF_KEY_FOR_PROFILE =>
-	FOR p in cdp_profile
-	    FILTER  p._key == @profileId
-	   	LET newAuthorizedViewers = APPEND(p.authorizedViewers, @authorizedViewers)
-	    LET newAuthorizedEditors = APPEND(p.authorizedEditors, @authorizedEditors)
-	    UPDATE p WITH { 
-	    	inSegments: [ {id: @segmentId, name: @segmentName, indexScore: @segmentIndexScore, queryHashedId: @queryHashedId} ],
-	    	authorizedViewers : newAuthorizedViewers, authorizedEditors : newAuthorizedEditors
-	    } IN cdp_profile
+	LET newSegment = {
+	  "id": @segmentId,
+	  "indexScore": @segmentIndexScore,
+	  "lastDataSynch": 0,
+	  "name": @segmentName,
+	  "queryHashedId": @queryHashedId,
+	  "authorizedViewers": @authorizedViewers,
+	  "authorizedEditors": @authorizedEditors
+	}
+
+	UPDATE { _key: @profileId } 
+	WITH { inSegments: [newSegment] } 
+	IN cdp_profile
 ;
 
 AQL_UPDATE_SEGMENT_REF_KEY_FOR_PROFILE =>
-	FOR p in cdp_profile
-	    FILTER p._key == @profileId
-	    LET alteredList = (
-	        FOR s IN p.inSegments
-	        LET newItem = ( s.id == @segmentId ? MERGE(s, {name : @segmentName, indexScore: @segmentIndexScore, queryHashedId: @queryHashedId }) : s )
-	        RETURN newItem
-	    )
-	   	LET newAuthorizedViewers = APPEND(p.authorizedViewers, @authorizedViewers)
-	    LET newAuthorizedEditors = APPEND(p.authorizedEditors, @authorizedEditors)
-	    UPDATE p WITH { inSegments: alteredList, authorizedViewers : newAuthorizedViewers, authorizedEditors : newAuthorizedEditors } IN cdp_profile
+	FOR profile IN cdp_profile
+	  FILTER profile._key == @profileId
+	  LET updatedSegments = (
+	    FOR segment IN profile.inSegments
+	      RETURN MERGE(
+	        segment,
+	        segment.id == @segmentId ? { 
+	          indexScore: @segmentIndexScore, 
+	          name: @segmentName,
+	          queryHashedId: @queryHashedId,
+	          authorizedViewers: @authorizedViewers,
+	          authorizedEditors: @authorizedEditors
+	        } : {}
+	      )
+	  )
+	  UPDATE profile WITH { inSegments: updatedSegments } IN cdp_profile
 ;
 
 AQL_APPEND_SEGMENT_REF_KEY_FOR_PROFILE =>
-	FOR p in cdp_profile
-	    FILTER p._key == @profileId
-	    LET newInSegments = APPEND(p.inSegments, [ {id: @segmentId, name : @segmentName, indexScore: @segmentIndexScore, queryHashedId: @queryHashedId} ] )
-	    LET newAuthorizedViewers = APPEND(p.authorizedViewers, @authorizedViewers)
-	    LET newAuthorizedEditors = APPEND(p.authorizedEditors, @authorizedEditors)
-	    UPDATE p WITH { inSegments: newInSegments, authorizedViewers : newAuthorizedViewers, authorizedEditors : newAuthorizedEditors } IN cdp_profile    
+	LET newSegment = {
+	  "id": @segmentId,
+	  "indexScore": @segmentIndexScore,
+	  "lastDataSynch": 0,
+	  "name": @segmentName,
+	  "queryHashedId": @queryHashedId,
+	  "authorizedViewers": @authorizedViewers,
+	  "authorizedEditors": @authorizedEditors
+	}
+
+	FOR profile IN cdp_profile
+	  FILTER profile._key == @profileId
+	  UPDATE profile 
+	  WITH { inSegments: PUSH(profile.inSegments, newSegment) } 
+	  IN cdp_profile   
 ;
 
-AQL_DELETE_SEGMENT_REF_KEY_FOR_PROFILE =>
-	FOR p in cdp_profile
-		FILTER p._key == @profileId
-		LET matchedSegmentRefs = ( FOR s IN p.inSegments LET newItem = ( s.id != @segmentId ? s : 0 ) RETURN newItem )
-		LET finalSegmentRefs = ( FOR s in matchedSegmentRefs FILTER s != 0 RETURN s)     
-		LET newViewers = ( FOR u IN p.authorizedViewers LET newItem = ( u NOT IN @oldAuthorizedViewers ? u : 0 ) RETURN newItem )
-		LET finalViewers = ( FOR u in newViewers FILTER u != 0 RETURN u) 
-		LET newEditors = ( FOR u IN p.authorizedEditors LET newItem = ( u NOT IN @oldAuthorizedEditors ? u : 0 ) RETURN newItem )
-		LET finalEditors = ( FOR u in newEditors FILTER u != 0 RETURN u) 
-		UPDATE p WITH { inSegments:  finalSegmentRefs, authorizedViewers: finalViewers, authorizedEditors: finalEditors } IN cdp_profile
+AQL_REMOVE_SEGMENT_REF_KEY_FOR_PROFILE =>
+	FOR profile IN cdp_profile
+	  FILTER profile._key == @profileId
+	  LET updatedSegments = (
+	    FOR segment IN profile.inSegments
+	      FILTER segment.id NOT IN @removedSegmentIds
+	      RETURN segment
+	  )
+  	UPDATE profile WITH { inSegments: updatedSegments } IN cdp_profile
 ;
 
 AQL_REMOVE_VIEWABLE_PROFILES_FOR_USER =>
@@ -1859,6 +1901,12 @@ AQL_GET_ALL_SEGMENTS =>
 		RETURN d
 ;
 
+AQL_GET_ALL_ACTIVE_SEGMENTS =>
+	FOR d in cdp_segment  
+		FILTER d.status > 0 
+		RETURN d
+;
+
 AQL_UPDATE_SEGMENT_TOTAL_COUNT =>
 	FOR d IN cdp_segment
 	    FILTER d._key == @segmentId