Fix sync bugs

Signed-off-by: Daishan Peng <[email protected]>
gptscript-ai · Sep 16, 2024 · d4d6931 · d4d6931
1 parent 12db481
commit d4d6931
Show file tree

Hide file tree

Showing 2 changed files with 60 additions and 34 deletions.
diff --git a/dist/index.js b/dist/index.js
@@ -95,17 +95,18 @@ function main() {
         // Fetch all pages
         const pages = yield fetchAllPages();
         let metadata = new Map();
-        const outputDir = path_1.default.join(process.env.WORKSPACE_DIR, 'knowledge', 'integrations', 'notion');
+        const outputDir = path_1.default.join(process.env.WORKSPACE_DIR, "knowledge", "integrations", "notion");
         yield (0, promises_1.mkdir)(outputDir, { recursive: true });
-        const metadataPath = path_1.default.join(outputDir, 'metadata.json');
+        const metadataPath = path_1.default.join(outputDir, "metadata.json");
         if (fs.existsSync(metadataPath)) {
-            metadata = new Map(Object.entries(JSON.parse(fs.readFileSync(metadataPath, 'utf8').toString())));
+            metadata = new Map(Object.entries(JSON.parse(fs.readFileSync(metadataPath, "utf8").toString())));
         }
         let updatedPages = 0;
         for (const page of pages) {
             if (metadata.has(page.id)) {
                 const entry = metadata.get(page.id);
-                if ((entry === null || entry === void 0 ? void 0 : entry.updatedAt) === page.last_edited_time && fs.existsSync(getPath(outputDir, page))) {
+                if ((entry === null || entry === void 0 ? void 0 : entry.updatedAt) === page.last_edited_time &&
+                    fs.existsSync(getPath(outputDir, page))) {
                     continue;
                 }
                 if (entry === null || entry === void 0 ? void 0 : entry.sync) {
@@ -119,12 +120,14 @@ function main() {
                     sync: (_a = entry === null || entry === void 0 ? void 0 : entry.sync) !== null && _a !== void 0 ? _a : false,
                 });
             }
-            metadata.set(page.id, {
-                url: page.url,
-                filename: path_1.default.basename(getPath(outputDir, page)),
-                updatedAt: page.last_edited_time,
-                sync: false,
-            });
+            else {
+                metadata.set(page.id, {
+                    url: page.url,
+                    filename: path_1.default.basename(getPath(outputDir, page)),
+                    updatedAt: page.last_edited_time,
+                    sync: false,
+                });
+            }
         }
         for (const [key, _] of metadata) {
             if (!pages.find((page) => page.id === key)) {
@@ -133,7 +136,7 @@ function main() {
                 metadata.delete(key);
             }
         }
-        yield (0, promises_1.writeFile)(metadataPath, JSON.stringify(Object.fromEntries(metadata)), 'utf8');
+        yield (0, promises_1.writeFile)(metadataPath, JSON.stringify(Object.fromEntries(metadata)), "utf8");
         console.log(`Finished writing ${updatedPages} pages to ${outputDir}`);
     });
 }

diff --git a/src/index.ts b/src/index.ts
@@ -22,14 +22,18 @@ async function main() {
     const pageContent = await getPageContent(notion, pageId);
     const fileDir = path.join(directory, pageId.toString());
     await mkdir(fileDir, { recursive: true });
-    const filePath = getPath(directory, page)
+    const filePath = getPath(directory, page);
     fs.writeFileSync(filePath, pageContent, "utf8");
   }
 
   function getPath(directory: string, page: PageObjectResponse): string {
     const pageId = page.id;
     const fileDir = path.join(directory, pageId.toString());
-    let title = ((page.properties?.title ?? page.properties?.Name) as any)?.title[0]?.plain_text?.trim().replaceAll(/\//g, "-");
+    let title = (
+      (page.properties?.title ?? page.properties?.Name) as any
+    )?.title[0]?.plain_text
+      ?.trim()
+      .replaceAll(/\//g, "-");
     if (!title) {
       title = pageId.toString();
     }
@@ -41,7 +45,6 @@ async function main() {
     let pages: any[] = [];
     let cursor: string | undefined = undefined;
 
-
     while (true) {
       const response: SearchResponse = await notion.search({
         filter: {
@@ -65,43 +68,59 @@ async function main() {
 
   // Fetch all pages
   const pages = await fetchAllPages();
-  let metadata: Map<string, {
-    url: string;
-    filename: string
-    updatedAt: string;
-    sync: boolean;
-  }> = new Map();
-  const outputDir = path.join(process.env.WORKSPACE_DIR!!, 'knowledge', 'integrations', 'notion');
+  let metadata: Map<
+    string,
+    {
+      url: string;
+      filename: string;
+      updatedAt: string;
+      sync: boolean;
+    }
+  > = new Map();
+  const outputDir = path.join(
+    process.env.WORKSPACE_DIR!!,
+    "knowledge",
+    "integrations",
+    "notion"
+  );
   await mkdir(outputDir, { recursive: true });
-  const metadataPath = path.join(outputDir, 'metadata.json');
+  const metadataPath = path.join(outputDir, "metadata.json");
   if (fs.existsSync(metadataPath)) {
-    metadata = new Map(Object.entries(JSON.parse(fs.readFileSync(metadataPath, 'utf8').toString())));
+    metadata = new Map(
+      Object.entries(
+        JSON.parse(fs.readFileSync(metadataPath, "utf8").toString())
+      )
+    );
   }
 
   let updatedPages = 0;
   for (const page of pages) {
     if (metadata.has(page.id)) {
-      const entry = metadata.get(page.id)
-      if (entry?.updatedAt === page.last_edited_time && fs.existsSync(getPath(outputDir, page))) {
+      const entry = metadata.get(page.id);
+      if (
+        entry?.updatedAt === page.last_edited_time &&
+        fs.existsSync(getPath(outputDir, page))
+      ) {
         continue;
       }
       if (entry?.sync) {
-        updatedPages++
+        updatedPages++;
         await writePageToFile(page, outputDir);
       }
       metadata.set(page.id, {
         url: page.url,
         filename: path.basename(getPath(outputDir, page)),
         updatedAt: page.last_edited_time,
         sync: entry?.sync ?? false,
-      })
+      });
+    } else {
+      metadata.set(page.id, {
+        url: page.url,
+        filename: path.basename(getPath(outputDir, page)),
+        updatedAt: page.last_edited_time,
+        sync: false,
+      });
     }
-    metadata.set(page.id, {
-      url: page.url,
-      filename: path.basename(getPath(outputDir, page)),
-      updatedAt: page.last_edited_time,
-      sync: false,
-    })
   }
 
   for (const [key, _] of metadata) {
@@ -112,7 +131,11 @@ async function main() {
     }
   }
 
-  await writeFile(metadataPath, JSON.stringify(Object.fromEntries(metadata)), 'utf8');
+  await writeFile(
+    metadataPath,
+    JSON.stringify(Object.fromEntries(metadata)),
+    "utf8"
+  );
 
   console.log(`Finished writing ${updatedPages} pages to ${outputDir}`);
 }