From ff60e6a621f6126cc994f1ba9f14ad6f8a630033 Mon Sep 17 00:00:00 2001
From: Daishan Peng <daishan@acorn.io>
Date: Tue, 3 Sep 2024 10:25:05 -0700
Subject: [PATCH] Only update files that have been added, fix table result

Signed-off-by: Daishan Peng <daishan@acorn.io>
---
 src/index.ts | 25 ++++++++++++++++++++++---
 src/page.ts  | 10 ++++++----
 2 files changed, 28 insertions(+), 7 deletions(-)

diff --git a/src/index.ts b/src/index.ts
index 612ac9c..4b9f1d6 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -58,22 +58,41 @@ async function main() {
   const pages = await fetchAllPages();
   let metadata: Map<string, {
     url: string;
+    updatedAt: string;
   }> = new Map();
-
   const outputDir = path.join(process.env.WORKSPACE_DIR!!, 'knowledge', 'integrations', 'notion');
   await mkdir(outputDir, { recursive: true });
+  const metadataPath = path.join(outputDir, 'metadata.json');
+  if (fs.existsSync(metadataPath)) {
+    metadata = new Map(Object.entries(JSON.parse(fs.readFileSync(metadataPath, 'utf8').toString())));
+  }
 
+  let updatedPages = 0;
   for (const page of pages) {
+    if (metadata.has(page.id)) {
+      if (metadata.get(page.id)!.updatedAt === page.last_edited_time) {
+        continue;
+      }
+    }
     await writePageToFile(page, outputDir);
     metadata.set(page.id, {
       url: page.url,
+      updatedAt: page.last_edited_time,
     })
+    updatedPages++
+  }
+
+  for (const [key, _] of metadata) {
+    if (!pages.find((page) => page.id === key)) {
+      fs.rmSync(path.join(outputDir, key), { recursive: true });
+      console.log(`Removed ${key} from ${outputDir}`);
+      metadata.delete(key);
+    }
   }
 
-  const metadataPath = path.join(outputDir, 'metadata.json');
   await writeFile(metadataPath, JSON.stringify(Object.fromEntries(metadata)), 'utf8');
 
-  console.log(`Finished writing ${pages.length} pages to ${outputDir}`);
+  console.log(`Finished writing ${updatedPages} pages to ${outputDir}`);
 }
 
 main()
diff --git a/src/page.ts b/src/page.ts
index 011e204..6aa3fa4 100644
--- a/src/page.ts
+++ b/src/page.ts
@@ -142,18 +142,20 @@ function fileToString(prefix: any, file: any) {
   return result
 }
 
-async function printTable(client: Client, table: any) {
+async function printTable(client: Client, table: any): Promise<string> {
+  let result = ""
   const children = await client.blocks.children.list({block_id: table.id})
   if (table.table.has_column_header && children.results.length > 0) {
-    printTableRow((children.results[0] as any).table_row, table.table.has_row_header, true)
+    result += printTableRow((children.results[0] as any).table_row, table.table.has_row_header, true)
     for (let i = 1; i < children.results.length; i++) {
-      printTableRow((children.results[i] as any).table_row, table.table.has_row_header, false)
+      result += printTableRow((children.results[i] as any).table_row, table.table.has_row_header, false)
     }
   } else {
     for (let r of children.results) {
-      printTableRow((r as any).table_row, table.table.has_row_header, false)
+      result += printTableRow((r as any).table_row, table.table.has_row_header, false)
     }
   }
+  return result;
 }
 
 function printTableRow(row: any, boldFirst: any, boldAll: any): string {