From 3d922fee3904bf7756edbc6b6c4d55b10ff94025 Mon Sep 17 00:00:00 2001
From: David Weiss <DavidWeiss2@gmail.com>
Date: Sun, 31 Dec 2023 23:37:41 +0200
Subject: [PATCH] Refactor Typesense import and upsert methods

---
 functions/src/backfillToTypesenseFromFirestore.js | 8 +++++---
 functions/src/indexToTypesenseOnFirestoreWrite.js | 7 +++----
 functions/src/utils.js                            | 7 +++++++
 3 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/functions/src/backfillToTypesenseFromFirestore.js b/functions/src/backfillToTypesenseFromFirestore.js
index a6b8993..3c62acb 100644
--- a/functions/src/backfillToTypesenseFromFirestore.js
+++ b/functions/src/backfillToTypesenseFromFirestore.js
@@ -43,7 +43,7 @@ module.exports = functions.firestore.document(config.typesenseBackfillTriggerDoc
       const typesense = createTypesenseClient();
 
       const querySnapshot =
-        await admin.firestore().collection(config.firestoreCollectionPath).get();
+      await admin.firestore().collection(config.firestoreCollectionPath).get();
       let currentDocumentNumber = 0;
       let currentDocumentsBatch = [];
       for (const firestoreDocument of querySnapshot.docs) {
@@ -55,7 +55,9 @@ module.exports = functions.firestore.document(config.typesenseBackfillTriggerDoc
             await typesense
                 .collections(encodeURIComponent(config.typesenseCollectionName))
                 .documents()
-                .import(currentDocumentsBatch);
+                .import(currentDocumentsBatch,
+                    utils.importDocumentsConfig(currentDocumentsBatch),
+                );
             currentDocumentsBatch = [];
             functions.logger.info(`Imported ${currentDocumentNumber} documents into Typesense`);
           } catch (error) {
@@ -68,7 +70,7 @@ module.exports = functions.firestore.document(config.typesenseBackfillTriggerDoc
           await typesense
               .collections(encodeURIComponent(config.typesenseCollectionName))
               .documents()
-              .import(currentDocumentsBatch);
+              .import(utils.importDocumentsConfig(currentDocumentsBatch));
           functions.logger.info(`Imported ${currentDocumentNumber} documents into Typesense`);
         } catch (error) {
           functions.logger.error("Import error", error);
diff --git a/functions/src/indexToTypesenseOnFirestoreWrite.js b/functions/src/indexToTypesenseOnFirestoreWrite.js
index ea80b4e..a8d3b5d 100644
--- a/functions/src/indexToTypesenseOnFirestoreWrite.js
+++ b/functions/src/indexToTypesenseOnFirestoreWrite.js
@@ -8,7 +8,7 @@ module.exports = functions.firestore.document(config.firestoreCollectionPath)
       const typesense = createTypesenseClient();
 
       if (snapshot.after.data() == null) {
-        // Delete
+      // Delete
         const documentId = snapshot.before.id;
         functions.logger.debug(`Deleting document ${documentId}`);
         return await typesense
@@ -16,7 +16,7 @@ module.exports = functions.firestore.document(config.firestoreCollectionPath)
             .documents(encodeURIComponent(documentId))
             .delete();
       } else {
-        // Create / update
+      // Create / update
 
         // snapshot.after.ref.get() will refetch the latest version of the document
         const latestSnapshot = await snapshot.after.ref.get();
@@ -25,7 +25,6 @@ module.exports = functions.firestore.document(config.firestoreCollectionPath)
         functions.logger.debug(`Upserting document ${JSON.stringify(typesenseDocument)}`);
         return await typesense
             .collections(encodeURIComponent(config.typesenseCollectionName))
-            .documents()
-            .upsert(typesenseDocument);
+            .documents().upsert(typesenseDocument, {"action": "emplace", "dirty_values": "coerce_or_drop"});
       }
     });
diff --git a/functions/src/utils.js b/functions/src/utils.js
index 498cbf6..6052c08 100644
--- a/functions/src/utils.js
+++ b/functions/src/utils.js
@@ -49,3 +49,10 @@ exports.typesenseDocumentFromSnapshot = async (
 
   return typesenseDocument;
 };
+
+exports.importDocumentsConfig = (batchSize) => ({
+  action: "emplace",
+  dirty_values: "coerce_or_drop",
+  return_doc: false,
+  batch_size: batchSize.length,
+});