From 0fec3f802e46fb8873eaa2ffcb727618b14b26e7 Mon Sep 17 00:00:00 2001 From: Daishan Peng Date: Fri, 6 Sep 2024 02:48:41 -0700 Subject: [PATCH] Add onedrive integration Signed-off-by: Daishan Peng --- actions/knowledge/filehelper.ts | 6 +- actions/knowledge/knowledge.ts | 8 +- actions/knowledge/notion.ts | 58 +------ actions/knowledge/onedrive.ts | 45 ++++++ actions/knowledge/tool.ts | 62 ++++++++ actions/knowledge/util.ts | 2 +- components/edit/configure.tsx | 8 +- components/knowledge/FileModal.tsx | 58 ++++++- components/knowledge/Notion.tsx | 53 ++++-- components/knowledge/OneDrive.tsx | 248 +++++++++++++++++++++++++++++ public/onedrive.svg | 1 + 11 files changed, 464 insertions(+), 85 deletions(-) create mode 100644 actions/knowledge/onedrive.ts create mode 100644 actions/knowledge/tool.ts create mode 100644 components/knowledge/OneDrive.tsx create mode 100644 public/onedrive.svg diff --git a/actions/knowledge/filehelper.ts b/actions/knowledge/filehelper.ts index e36200bb..dfd9dd84 100644 --- a/actions/knowledge/filehelper.ts +++ b/actions/knowledge/filehelper.ts @@ -29,13 +29,9 @@ export async function getFileOrFolderSizeInKB( return 0; } -export async function getBasename(filePath: string): Promise { - return path.basename(filePath); -} - export async function importFiles( files: string[], - type: 'local' | 'notion' + type: 'local' | 'notion' | 'onedrive' ): Promise> { const result: Map = new Map(); diff --git a/actions/knowledge/knowledge.ts b/actions/knowledge/knowledge.ts index 629a81b6..7729c82f 100644 --- a/actions/knowledge/knowledge.ts +++ b/actions/knowledge/knowledge.ts @@ -65,7 +65,7 @@ export async function ensureFiles( if (!fs.existsSync(filePath)) { if (file[1].type === 'local') { await fs.promises.copyFile(file[0], filePath); - } else if (file[1].type === 'notion') { + } else if (file[1].type === 'notion' || file[1].type === 'onedrive') { if ( fs.existsSync(filePath) && fs.lstatSync(filePath).isSymbolicLink() @@ -78,7 +78,7 @@ export async function ensureFiles( } if (!updateOnly) { - for (const type of ['local', 'notion']) { + for (const type of ['local', 'notion', 'onedrive']) { if (!fs.existsSync(path.join(dir, type))) { continue; } @@ -124,7 +124,7 @@ export async function getFiles( if (!fs.existsSync(dir)) { return result; } - for (const type of ['local', 'notion']) { + for (const type of ['local', 'notion', 'onedrive']) { if (!fs.existsSync(path.join(dir, type))) { continue; } @@ -135,7 +135,7 @@ export async function getFiles( filePath = await fs.promises.readlink(filePath); } result.set(filePath, { - type: type as 'local' | 'notion', + type: type as any, fileName: file, size: await getFileOrFolderSizeInKB(path.join(dir, type, file)), }); diff --git a/actions/knowledge/notion.ts b/actions/knowledge/notion.ts index c11a6f12..f1825d0a 100644 --- a/actions/knowledge/notion.ts +++ b/actions/knowledge/notion.ts @@ -3,12 +3,7 @@ import fs from 'fs'; import path from 'path'; import { WORKSPACE_DIR } from '@/config/env'; -import { - GPTScript, - PromptFrame, - Run, - RunEventType, -} from '@gptscript-ai/gptscript'; +import { runSyncTool } from '@/actions/knowledge/tool'; export async function isNotionConfigured() { return fs.existsSync( @@ -22,37 +17,15 @@ export async function isNotionConfigured() { ); } -function readFilesRecursive(dir: string): string[] { - let results: string[] = []; - - const list = fs.readdirSync(dir); - list.forEach((file) => { - if (file === 'metadata.json') return; - const filePath = path.join(dir, file); - const stat = fs.statSync(filePath); - - if (stat && stat.isDirectory()) { - // Recursively read the directory - results = results.concat(readFilesRecursive(filePath)); - } else { - // Add the file path to the results - results.push(filePath); - } - }); - - return results; -} - export async function getNotionFiles(): Promise< Map > { const dir = path.join(WORKSPACE_DIR(), 'knowledge', 'integrations', 'notion'); - const filePaths = readFilesRecursive(dir); const metadataFromFiles = fs.readFileSync(path.join(dir, 'metadata.json')); const metadata = JSON.parse(metadataFromFiles.toString()); const result = new Map(); - for (const filePath of filePaths) { - const pageID = path.basename(path.dirname(filePath)); + for (const pageID in metadata) { + const filePath = path.join(dir, pageID, metadata[pageID].filename); result.set(filePath, { url: metadata[pageID].url, fileName: path.basename(filePath), @@ -63,28 +36,5 @@ export async function getNotionFiles(): Promise< } export async function runNotionSync(authed: boolean): Promise { - const gptscript = new GPTScript({ - DefaultModelProvider: 'github.com/gptscript-ai/gateway-provider', - }); - - const runningTool = await gptscript.run( - 'github.com/gptscript-ai/knowledge-notion-integration', - { - prompt: true, - } - ); - if (!authed) { - const handlePromptEvent = (runningTool: Run) => { - return new Promise((resolve) => { - runningTool.on(RunEventType.Prompt, (data: PromptFrame) => { - resolve(data.id); - }); - }); - }; - - const id = await handlePromptEvent(runningTool); - await gptscript.promptResponse({ id, responses: {} }); - } - await runningTool.text(); - return; + return runSyncTool(authed, 'notion'); } diff --git a/actions/knowledge/onedrive.ts b/actions/knowledge/onedrive.ts new file mode 100644 index 00000000..fae5c2d2 --- /dev/null +++ b/actions/knowledge/onedrive.ts @@ -0,0 +1,45 @@ +'use server'; +import fs from 'fs'; +import path from 'path'; +import { WORKSPACE_DIR } from '@/config/env'; +import { runSyncTool } from '@/actions/knowledge/tool'; + +export async function isOneDriveConfigured() { + return fs.existsSync( + path.join( + WORKSPACE_DIR(), + 'knowledge', + 'integrations', + 'onedrive', + 'metadata.json' + ) + ); +} + +export async function getOneDriveFiles(): Promise< + Map +> { + const dir = path.join( + WORKSPACE_DIR(), + 'knowledge', + 'integrations', + 'onedrive' + ); + const metadataFromFiles = fs.readFileSync(path.join(dir, 'metadata.json')); + const metadata = JSON.parse(metadataFromFiles.toString()); + const result = new Map(); + for (const documentID in metadata) { + result.set(path.join(dir, documentID, metadata[documentID].fileName), { + url: metadata[documentID].url, + fileName: metadata[documentID].fileName, + }); + } + return result; +} + +// syncFiles syncs all files only when they are selected +// todo: we can stop syncing once file is no longer used by any other script + +export async function runOneDriveSync(authed: boolean): Promise { + return runSyncTool(authed, 'onedrive'); +} diff --git a/actions/knowledge/tool.ts b/actions/knowledge/tool.ts new file mode 100644 index 00000000..c7f58a92 --- /dev/null +++ b/actions/knowledge/tool.ts @@ -0,0 +1,62 @@ +'use server'; + +import { + GPTScript, + PromptFrame, + Run, + RunEventType, +} from '@gptscript-ai/gptscript'; +import path from 'path'; +import { WORKSPACE_DIR } from '@/config/env'; +import fs from 'fs'; + +export async function runSyncTool( + authed: boolean, + tool: 'notion' | 'onedrive' +): Promise { + const gptscript = new GPTScript({ + DefaultModelProvider: 'github.com/gptscript-ai/gateway-provider', + }); + + let toolUrl = ''; + if (tool === 'notion') { + toolUrl = 'github.com/gptscript-ai/knowledge-notion-integration@46a273e'; + } else if (tool === 'onedrive') { + toolUrl = 'github.com/gptscript-ai/knowledge-onedrive-integration@a85a498'; + } + const runningTool = await gptscript.run(toolUrl, { + prompt: true, + }); + if (!authed) { + const handlePromptEvent = (runningTool: Run) => { + return new Promise((resolve) => { + runningTool.on(RunEventType.Prompt, (data: PromptFrame) => { + resolve(data.id); + }); + }); + }; + + const id = await handlePromptEvent(runningTool); + await gptscript.promptResponse({ id, responses: {} }); + } + await runningTool.text(); + return; +} + +export async function syncFiles( + selectedFiles: string[], + type: 'notion' | 'onedrive' +): Promise { + const dir = path.join(WORKSPACE_DIR(), 'knowledge', 'integrations', type); + const metadataFromFiles = fs.readFileSync(path.join(dir, 'metadata.json')); + const metadata = JSON.parse(metadataFromFiles.toString()); + for (const file of selectedFiles) { + const documentID = path.basename(path.dirname(file)); + const detail = metadata[documentID]; + detail.sync = true; + metadata[documentID] = detail; + } + fs.writeFileSync(path.join(dir, 'metadata.json'), JSON.stringify(metadata)); + await runSyncTool(true, type); + return; +} diff --git a/actions/knowledge/util.ts b/actions/knowledge/util.ts index 782b87b0..24ddf26a 100644 --- a/actions/knowledge/util.ts +++ b/actions/knowledge/util.ts @@ -1,7 +1,7 @@ export interface FileDetail { fileName: string; size: number; - type: 'local' | 'notion'; + type: 'local' | 'notion' | 'onedrive'; } export function gatewayTool(): string { diff --git a/components/edit/configure.tsx b/components/edit/configure.tsx index 4ee63a51..afa1ff57 100644 --- a/components/edit/configure.tsx +++ b/components/edit/configure.tsx @@ -31,6 +31,7 @@ import { RiFoldersLine } from 'react-icons/ri'; import FileModal from '@/components/knowledge/FileModal'; import { gatewayTool } from '@/actions/knowledge/util'; import { importFiles } from '@/actions/knowledge/filehelper'; +import { DiOnedrive } from 'react-icons/di'; interface ConfigureProps { collapsed?: boolean; @@ -191,14 +192,17 @@ const Configure: React.FC = ({ collapsed }) => { ([key, fileDetail], _) => (
-
+
{fileDetail.type === 'local' && ( )} {fileDetail.type === 'notion' && ( )} -
+ {fileDetail.type === 'onedrive' && ( + + )} +

{fileDetail.fileName}

diff --git a/components/knowledge/FileModal.tsx b/components/knowledge/FileModal.tsx index 7cc3b3f8..eeffbf7f 100644 --- a/components/knowledge/FileModal.tsx +++ b/components/knowledge/FileModal.tsx @@ -1,15 +1,20 @@ import { + Avatar, Button, Modal, ModalBody, ModalContent, useDisclosure, } from '@nextui-org/react'; -import { Image } from '@nextui-org/image'; import { BiPlus } from 'react-icons/bi'; import { NotionFileModal } from '@/components/knowledge/Notion'; import { isNotionConfigured, runNotionSync } from '@/actions/knowledge/notion'; import { useState } from 'react'; +import { OnedriveFileModal } from '@/components/knowledge/OneDrive'; +import { + isOneDriveConfigured, + runOneDriveSync, +} from '@/actions/knowledge/onedrive'; interface FileModalProps { isOpen: boolean; @@ -19,9 +24,12 @@ interface FileModalProps { const FileModal = ({ isOpen, onClose, handleAddFile }: FileModalProps) => { const notionModal = useDisclosure(); + const onedriveModal = useDisclosure(); const [isSyncing, setIsSyncing] = useState(false); const [notionConfigured, setNotionConfigured] = useState(false); const [notionSyncError, setNotionSyncError] = useState(''); + const [oneDriveConfigured, setOneDriveConfigured] = useState(false); + const [oneDriveSyncError, setOneDriveSyncError] = useState(''); const onClickNotion = async () => { onClose(); @@ -40,6 +48,23 @@ const FileModal = ({ isOpen, onClose, handleAddFile }: FileModalProps) => { } }; + const onClickOnedrive = async () => { + onClose(); + onedriveModal.onOpen(); + const isConfigured = await isOneDriveConfigured(); + if (!isConfigured) { + setIsSyncing(true); + try { + await runOneDriveSync(false); + setOneDriveConfigured(true); + } catch (e) { + setOneDriveSyncError((e as Error).toString()); + } finally { + setIsSyncing(false); + } + } + }; + return ( <> { onClick={onClickNotion} className="flex w-full items-center justify-center gap-3 rounded-md bg-white px-3 py-2 text-sm font-semibold text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 hover:bg-gray-50 focus-visible:ring-transparent hover:cursor-pointer" > - Notion Icon + Sync From Notion + @@ -86,6 +130,16 @@ const FileModal = ({ isOpen, onClose, handleAddFile }: FileModalProps) => { syncError={notionSyncError} setSyncError={setNotionSyncError} /> + ); }; diff --git a/components/knowledge/Notion.tsx b/components/knowledge/Notion.tsx index 584817b1..258f374e 100644 --- a/components/knowledge/Notion.tsx +++ b/components/knowledge/Notion.tsx @@ -1,4 +1,5 @@ import { + Avatar, Button, Input, Modal, @@ -15,7 +16,6 @@ import { } from '@nextui-org/react'; import React, { useContext, useEffect, useState } from 'react'; import { IoMdRefresh } from 'react-icons/io'; -import { Image } from '@nextui-org/image'; import { getNotionFiles, isNotionConfigured, @@ -25,6 +25,7 @@ import { CiSearch, CiShare1 } from 'react-icons/ci'; import { EditContext } from '@/contexts/edit'; import { importFiles } from '@/actions/knowledge/filehelper'; import { Link } from '@nextui-org/react'; +import { syncFiles } from '@/actions/knowledge/tool'; interface NotionFileModalProps { isOpen: boolean; @@ -49,6 +50,7 @@ export const NotionFileModal = ({ }: NotionFileModalProps) => { const { droppedFiles, setDroppedFiles } = useContext(EditContext); const [searchQuery, setSearchQuery] = useState(''); + const [importing, setImporting] = useState(false); const [notionFiles, setNotionFiles] = useState< Map @@ -79,19 +81,26 @@ export const NotionFileModal = ({ }, [isOpen, notionConfigured]); const onClickImport = async () => { - const files = await importFiles(selectedFile, 'notion'); - setDroppedFiles((prev) => { - const newMap = new Map(prev); - for (const [key, file] of Array.from(newMap.entries())) { - if (file.type === 'notion') { - newMap.delete(key); + setImporting(true); + try { + await syncFiles(selectedFile, 'notion'); + const files = await importFiles(selectedFile, 'notion'); + setDroppedFiles((prev) => { + const newMap = new Map(prev); + for (const [key, file] of Array.from(newMap.entries())) { + if (file.type === 'notion') { + newMap.delete(key); + } + } + for (const [key, file] of Array.from(files.entries())) { + newMap.set(key, file); } - } - for (const [key, file] of Array.from(files.entries())) { - newMap.set(key, file); - } - return newMap; - }); + return newMap; + }); + } finally { + setImporting(false); + } + onClose(); }; @@ -131,7 +140,12 @@ export const NotionFileModal = ({
- Notion Icon +

Notion

@@ -175,7 +189,7 @@ export const NotionFileModal = ({ }} > - NAME + Name Link @@ -193,7 +207,7 @@ export const NotionFileModal = ({

{value.fileName}

- + diff --git a/components/knowledge/OneDrive.tsx b/components/knowledge/OneDrive.tsx new file mode 100644 index 00000000..142843ba --- /dev/null +++ b/components/knowledge/OneDrive.tsx @@ -0,0 +1,248 @@ +import { + Avatar, + Button, + Input, + Modal, + ModalBody, + ModalContent, + ModalFooter, + ModalHeader, + Table, + TableBody, + TableCell, + TableColumn, + TableHeader, + TableRow, +} from '@nextui-org/react'; +import React, { useContext, useEffect, useState } from 'react'; +import { IoMdRefresh } from 'react-icons/io'; +import { + getOneDriveFiles, + isOneDriveConfigured, + runOneDriveSync, +} from '@/actions/knowledge/onedrive'; +import { CiSearch, CiShare1 } from 'react-icons/ci'; +import { EditContext } from '@/contexts/edit'; +import { importFiles } from '@/actions/knowledge/filehelper'; +import { Link } from '@nextui-org/react'; +import { syncFiles } from '@/actions/knowledge/tool'; + +interface OnedriveFileModalProps { + isOpen: boolean; + onClose: () => void; + isSyncing: boolean; + setIsSyncing: React.Dispatch>; + onedriveConfigured: boolean; + setOnedriveConfigured: React.Dispatch>; + syncError: string; + setSyncError: React.Dispatch>; +} + +export const OnedriveFileModal = ({ + isOpen, + onClose, + onedriveConfigured, + setOnedriveConfigured, + isSyncing, + setIsSyncing, + syncError, + setSyncError, +}: OnedriveFileModalProps) => { + const { droppedFiles, setDroppedFiles } = useContext(EditContext); + const [searchQuery, setSearchQuery] = useState(''); + const [importing, setImporting] = useState(false); + + const [onedriveFiles, setOnedriveFiles] = useState< + Map + >(new Map()); + + const [selectedFile, setSelectedFile] = useState( + Array.from(droppedFiles.keys()) + ); + + useEffect(() => { + if (onedriveConfigured) return; + + const setConfigured = async () => { + setOnedriveConfigured(await isOneDriveConfigured()); + }; + setConfigured(); + }, [onedriveConfigured, isOpen]); + + useEffect(() => { + if (!onedriveConfigured) return; + + const setFiles = async () => { + const onedriveFiles = await getOneDriveFiles(); + setOnedriveFiles(onedriveFiles); + }; + + setFiles(); + }, [isOpen, onedriveConfigured]); + + const onClickImport = async () => { + setImporting(true); + try { + await syncFiles(selectedFile, 'onedrive'); + const files = await importFiles(selectedFile, 'onedrive'); + setDroppedFiles((prev) => { + const newMap = new Map(prev); + for (const [key, file] of Array.from(newMap.entries())) { + if (file.type === 'onedrive') { + newMap.delete(key); + } + } + for (const [key, file] of Array.from(files.entries())) { + newMap.set(key, file); + } + return newMap; + }); + } finally { + setImporting(false); + } + + onClose(); + }; + + const syncOnedrive = async () => { + setIsSyncing(true); + try { + const isConfigured = await isOneDriveConfigured(); + await runOneDriveSync(isConfigured); + setOnedriveConfigured(isConfigured); + setOnedriveFiles(await getOneDriveFiles()); + setSyncError(''); + } catch (e) { + setSyncError((e as Error).toString()); + } finally { + setIsSyncing(false); + } + }; + + const handleSelectedFileChange = (selected: any) => { + if (selected === 'all') { + setSelectedFile(Array.from(onedriveFiles.keys())); + } else { + setSelectedFile(Array.from(selected as Set)); + } + }; + + return ( + + + +
+
+ +

Onedrive

+
+ +
+ + {syncError && ( +

{syncError}

+ )} +
+
+ + + { + setSearchQuery(e.target.value); + }} + startContent={} + /> + {onedriveConfigured && onedriveFiles.size > 0 ? ( +
+ { + handleSelectedFileChange(selected); + }} + > + + Name + Link + + + {Array.from(onedriveFiles.entries()) + .filter(([_, file]) => { + if (!searchQuery) return true; + return file.fileName + .toLowerCase() + .includes(searchQuery.toLowerCase()); + }) + .map(([key, value]) => ( + + +
+

{value.fileName}

+
+ + +
+
+ ) : ( +
+

+ {`Click the "Sync" button to sync your Onedrive files`} +

+
+ )} + + + + + + + ); +}; diff --git a/public/onedrive.svg b/public/onedrive.svg new file mode 100644 index 00000000..16dc1815 --- /dev/null +++ b/public/onedrive.svg @@ -0,0 +1 @@ +OfficeCore10_32x_24x_20x_16x_01-22-2019