diff --git a/express/main.js b/express/main.js index 3121e14..b610934 100644 --- a/express/main.js +++ b/express/main.js @@ -3,7 +3,7 @@ import isDev from 'electron-is-dev' import settings from 'electron-settings'; import { server, getPort } from './src/app.js'; -import { initChatBookSetting, initChatBookDb } from './src/utils/db.js'; +import { initChatBookSetting, initChatBookDbExec } from './src/utils/db.js'; const PORT = getPort(); @@ -27,7 +27,7 @@ function createMainWindow() { ChatBookSetting = await settings.get('chatbook'); console.log("ChatBookSetting main.js", ChatBookSetting) await initChatBookSetting(ChatBookSetting); - await initChatBookDb(); + await initChatBookDbExec(); mainWindow.loadURL('http://localhost:' + PORT); }); diff --git a/express/package-lock.json b/express/package-lock.json index daaf890..113df72 100644 --- a/express/package-lock.json +++ b/express/package-lock.json @@ -5631,7 +5631,8 @@ }, "node_modules/fs": { "version": "0.0.1-security", - "license": "ISC" + "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz", + "integrity": "sha512-3XY9e1pP0CVEUCdj5BmfIZxRBTSDycnbqhIOGec9QYtmVH2fbLpj86CFWkrNOkt/Fvty4KZG5lTglL9j/gJ87w==" }, "node_modules/fs-constants": { "version": "1.0.0", diff --git a/express/src/router/user.js b/express/src/router/user.js index eb29905..799c758 100644 --- a/express/src/router/user.js +++ b/express/src/router/user.js @@ -1,7 +1,7 @@ // app.ts import express from 'express'; - import { checkUserPassword, registerUser, changeUserPasswordByToken, changeUserDetail, changeUserStatus, checkUserToken, getUsers, getUserLogsAll, getUserLogs, getOneUserByToken, getUserByEmail, updateUserImageFavorite, updateUserVideoFavorite, refreshUserToken, addUserAgent, deleteUserAgent, getUserAgents, getMyAgents, addUser, editUser, deleteUser } from '../utils/user.js'; + import { checkUserPassword, registerUser, changeUserPasswordByToken, changeUserDetail, changeUserStatus, checkUserToken, getUsers, getUserLogsAll, getUserLogs, getOneUserByToken, getUserByEmail, updateUserImageFavorite, updateUserVideoFavorite, refreshUserToken, addUser, editUser, deleteUser } from '../utils/user.js'; const app = express(); @@ -152,47 +152,4 @@ res.end(); }); - app.post('/api/user/agent/add', async (req, res) => { - const { authorization } = req.headers; - const addUserAgentData = await addUserAgent(authorization, req.body); - res.status(200).json(addUserAgentData); - res.end(); - }); - - app.post('/api/user/agent/delete', async (req, res) => { - const { authorization } = req.headers; - const deleteUserAgentData = await deleteUserAgent(authorization, req.body); - res.status(200).json(deleteUserAgentData); - res.end(); - }); - - app.post('/api/user/agents', async (req, res) => { - const { authorization } = req.headers; - const { pageid, pagesize } = req.body; - const checkUserTokenData = await checkUserToken(authorization); - if(checkUserTokenData && checkUserTokenData.data && checkUserTokenData.data.id && (checkUserTokenData.data.role == 'admin' || checkUserTokenData.data.role == 'user')) { - const getUserAgentsData = await getUserAgents(checkUserTokenData.data.id, Number(pageid), Number(pagesize)); - res.status(200).json(getUserAgentsData); - } - else { - res.status(200).json({"status":"error", "msg":"Token is invalid", "data": null}); - } - res.end(); - }); - - app.post('/api/my/agents', async (req, res) => { - const { authorization } = req.headers; - const { pageid, pagesize } = req.body; - const checkUserTokenData = await checkUserToken(authorization); - if(checkUserTokenData && checkUserTokenData.data && checkUserTokenData.data.id && (checkUserTokenData.data.role == 'admin' || checkUserTokenData.data.role == 'user')) { - const getMyAgentsData = await getMyAgents(checkUserTokenData.data.id, Number(pageid), Number(pagesize)); - res.status(200).json(getMyAgentsData); - } - else { - res.status(200).json({"status":"error", "msg":"Token is invalid", "data": null}); - } - res.end(); - }); - - export default app; diff --git a/express/src/router/utils.js b/express/src/router/utils.js index 7f7e2ee..726f035 100644 --- a/express/src/router/utils.js +++ b/express/src/router/utils.js @@ -4,7 +4,7 @@ import { MenuListAdmin, MenuListUser } from '../utils/const.js'; import { checkUserToken } from '../utils/user.js'; - import { getLogsPage, clearShortLogs, clearAllLogs, getTemplate, getLLMSSetting, getFilesPage, getFilesDatasetId, getChatLogByKnowledgeIdAndUserId, addKnowledge, setOpenAISetting, setTemplate, getKnowledgePage, wholeSiteStatics, getAllImages, deleteUserLogByKnowledgeId, getAgentsPage, getAgentsEnabledList, addAgent, editAgent } from '../utils/utils.js'; + import { getLogsPage, clearShortLogs, clearAllLogs, getTemplate, getLLMSSetting, getFilesPage, getFilesDatasetId, getChatLogByDatasetIdAndUserId, setOpenAISetting, setTemplate, wholeSiteStatics, getAllImages, deleteUserLogByDatasetId } from '../utils/utils.js'; const app = express(); @@ -34,27 +34,13 @@ res.end(); }); - app.post('/api/addknowledge', async (req, res) => { - const { authorization } = req.headers; - const checkUserTokenData = await checkUserToken(authorization); - if(checkUserTokenData && checkUserTokenData.data && checkUserTokenData.data.email && (checkUserTokenData.data.role == 'admin' || checkUserTokenData.data.role == 'user')) { - console.log("checkUserTokenDatacheckUserTokenDatacheckUserTokenDatacheckUserTokenData", checkUserTokenData) - const addKnowledgeData = await addKnowledge({...req.body, userId: checkUserTokenData.data.id}); - res.status(200).json(addKnowledgeData); - } - else { - res.status(200).json({"status":"error", "msg":"Token is invalid", "data": null}); - } - res.end(); - }); - app.get('/api/chatlog/agent/:datasetId/:userId/:pageid/:pagesize', async (req, res) => { const { datasetId, userId, pageid, pagesize } = req.params; const { authorization } = req.headers; const checkUserTokenData = await checkUserToken(authorization); if(checkUserTokenData && checkUserTokenData.data && checkUserTokenData.data.email && (checkUserTokenData.data.role == 'admin' || checkUserTokenData.data.role == 'user')) { - const getChatLogByKnowledgeIdAndUserIdData = await getChatLogByKnowledgeIdAndUserId(datasetId, Number(userId), Number(pageid), Number(pagesize)); - res.status(200).json(getChatLogByKnowledgeIdAndUserIdData); + const getChatLogByDatasetIdAndUserIdData = await getChatLogByDatasetIdAndUserId(datasetId, Number(userId), Number(pageid), Number(pagesize)); + res.status(200).json(getChatLogByDatasetIdAndUserIdData); } else { res.status(200).json({"status":"error", "msg":"Token is invalid", "data": null}); @@ -67,8 +53,8 @@ const { authorization } = req.headers; const checkUserTokenData = await checkUserToken(authorization); if(checkUserTokenData && checkUserTokenData.data && checkUserTokenData.data.email && (checkUserTokenData.data.role == 'admin' || checkUserTokenData.data.role == 'user')) { - const deleteUserLogByKnowledgeIdData = await deleteUserLogByKnowledgeId(datasetId, Number(userId)); - res.status(200).json(deleteUserLogByKnowledgeIdData); + const deleteUserLogByDatasetIdData = await deleteUserLogByDatasetId(datasetId, Number(userId)); + res.status(200).json(deleteUserLogByDatasetIdData); } else { res.status(200).json({"status":"error", "msg":"Token is invalid", "data": null}); @@ -132,20 +118,6 @@ res.end(); }); - app.get('/api/knowledge/:pageid/:pagesize', async (req, res) => { - const { pageid, pagesize } = req.params; - const { authorization } = req.headers; - const checkUserTokenData = await checkUserToken(authorization); - if(checkUserTokenData && checkUserTokenData.data && checkUserTokenData.data.email && (checkUserTokenData.data.role == 'admin' || checkUserTokenData.data.role == 'user')) { - const getKnowledgePageData = await getKnowledgePage(Number(pageid), Number(pagesize)); - res.status(200).json(getKnowledgePageData); - } - else { - res.status(200).json({"status":"error", "msg":"Token is invalid", "data": null}); - } - res.end(); - }); - app.get('/api/logs/:pageid/:pagesize', async (req, res) => { const { pageid, pagesize } = req.params; const { authorization } = req.headers; @@ -233,75 +205,4 @@ res.status(200).json(generateimageData).end(); }); - app.get('/api/agentsall/:pageid/:pagesize', async (req, res) => { - const { pageid, pagesize } = req.params; - const { authorization } = req.headers; - const checkUserTokenData = await checkUserToken(authorization); - if(checkUserTokenData && checkUserTokenData.data && checkUserTokenData.data.email && (checkUserTokenData.data.role == 'admin')) { - const getAgentsPageData = await getAgentsPage(Number(pageid), Number(pagesize)); - res.status(200).json(getAgentsPageData); - } - else { - res.status(200).json({"status":"error", "msg":"Token is invalid", "data": null}); - } - res.end(); - }); - - app.post('/api/agents/:pageid/:pagesize', async (req, res) => { - const { pageid, pagesize } = req.params; - const { authorization } = req.headers; - const { type, search } = req.body; - const checkUserTokenData = await checkUserToken(authorization); - if(checkUserTokenData && checkUserTokenData.data && checkUserTokenData.data.email && (checkUserTokenData.data.role == 'admin' || checkUserTokenData.data.role == 'user')) { - const getAgentsPageData = await getAgentsEnabledList(Number(pageid), Number(pagesize), type, search); - res.status(200).json(getAgentsPageData); - } - else { - res.status(200).json({"status":"error", "msg":"Token is invalid", "data": null}); - } - res.end(); - }); - - app.post('/api/addagent', async (req, res) => { - const { authorization } = req.headers; - const checkUserTokenData = await checkUserToken(authorization); - if(checkUserTokenData && checkUserTokenData.data && checkUserTokenData.data.email && (checkUserTokenData.data.role == 'admin' || checkUserTokenData.data.role == 'user')) { - console.log("checkUserTokenData addagent", checkUserTokenData) - const addAgentData = await addAgent({...req.body, userId: checkUserTokenData.data.id}); - res.status(200).json(addAgentData); - } - else { - res.status(200).json({"status":"error", "msg":"Token is invalid", "data": null}); - } - res.end(); - }); - - app.post('/api/editagent', async (req, res) => { - const { authorization } = req.headers; - const checkUserTokenData = await checkUserToken(authorization); - if(checkUserTokenData && checkUserTokenData.data && checkUserTokenData.data.email && (checkUserTokenData.data.role == 'admin' || checkUserTokenData.data.role == 'user')) { - console.log("checkUserTokenData editagent", checkUserTokenData) - const editAgentData = await editAgent({...req.body, userId: checkUserTokenData.data.id}); - res.status(200).json(editAgentData); - } - else { - res.status(200).json({"status":"error", "msg":"Token is invalid", "data": null}); - } - res.end(); - }); - - app.post('/api/deleteagent', async (req, res) => { - const { authorization } = req.headers; - const checkUserTokenData = await checkUserToken(authorization); - if(checkUserTokenData && checkUserTokenData.data && checkUserTokenData.data.email && (checkUserTokenData.data.role == 'admin' || checkUserTokenData.data.role == 'user')) { - console.log("checkUserTokenData deleteagent", checkUserTokenData) - const editAgentData = await editAgent({...req.body, userId: checkUserTokenData.data.id}); - res.status(200).json(editAgentData); - } - else { - res.status(200).json({"status":"error", "msg":"Token is invalid", "data": null}); - } - res.end(); - }); - export default app; diff --git a/express/src/utils/app.js b/express/src/utils/app.js index 79d7cf5..d620420 100644 --- a/express/src/utils/app.js +++ b/express/src/utils/app.js @@ -1,17 +1,10 @@ -import * as fs from 'fs' -import multer from 'multer' -import path from 'path' -import * as crypto from 'crypto' -import validator from 'validator'; -import { promisify } from 'util'; -import { DataDir, CONDENSE_TEMPLATE_INIT, QA_TEMPLATE_INIT } from './const.js'; - -import { db, getDbRecord, getDbRecordALL } from './db.js' -import { filterString, log, formatDate, getNanoid } from './utils.js' - +import { initChatBookDb } from './db.js' +import { filterString, log, getNanoid } from './utils.js' export async function addApp(Params) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + try{ console.log("ParamsParamsParamsParamsParams", Params) Params._id = filterString(Params._id) @@ -39,6 +32,8 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } export async function editApp(Params) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + try{ Params._id = filterString(Params._id) Params.teamId = filterString(Params.teamId) @@ -65,6 +60,8 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } export async function editAppById(Params) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + try{ Params._id = filterString(Params._id) Params.teamId = filterString(Params.teamId) @@ -90,6 +87,8 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } export async function deleteApp(Params) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + try{ Params._id = filterString(Params.appId) Params.userId = filterString(Params.userId) @@ -106,6 +105,8 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } export async function deleteAppById(Params) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + try{ Params._id = filterString(Params._id) Params.id = filterString(Params.id) @@ -122,6 +123,8 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } export async function getApp(id, userId) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const idFilter = filterString(id) const SettingRS = await (getDbRecordALL)(`SELECT data, avatar, name, intro, type, groupTwo, permission from app where _id = ? `, [idFilter]); @@ -138,6 +141,8 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } export async function getAppById(_id, id) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const _idFilter = filterString(_id) const idFilter = filterString(id) @@ -155,6 +160,8 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } export async function getAppByPublishId(id) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const idFilter = filterString(id) const PublishApp = await (getDbRecordALL)(`SELECT * from publish where _id = ?`, [idFilter]); @@ -176,6 +183,8 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } export async function getAppPage(pageid, pagesize, userId) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const pageidFiler = Number(pageid) < 0 ? 0 : pageid; const pagesizeFiler = Number(pagesize) < 5 ? 5 : pagesize; const From = pageidFiler * pagesizeFiler; @@ -198,6 +207,8 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } export async function getAppPageAll(pageid, pagesize, data) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const pageidFiler = Number(pageid) < 0 ? 0 : pageid; const pagesizeFiler = Number(pagesize) < 5 ? 5 : pagesize; const From = pageidFiler * pagesizeFiler; @@ -240,6 +251,8 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } export async function getChatlogPageByApp(appId, pageid, pagesize, userId) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const appIdFileter = filterString(appId) const pageidFiler = Number(pageid) < 0 ? 0 : pageid; const pagesizeFiler = Number(pagesize) < 5 ? 5 : pagesize; @@ -263,6 +276,8 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } export async function getChatlogStaticPageByApp(appId, pageid, pagesize) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const appIdFileter = filterString(appId) const pageidFiler = Number(pageid) < 0 ? 0 : pageid; const pagesizeFiler = Number(pagesize) < 5 ? 5 : pagesize; @@ -299,6 +314,8 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } export async function getPublishsPageByApp(appId, pageid, pagesize, userId) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const appIdFileter = filterString(appId) const pageidFiler = Number(pageid) < 0 ? 0 : pageid; const pagesizeFiler = Number(pagesize) < 5 ? 5 : pagesize; @@ -322,6 +339,8 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } export async function getPublishsAll(pageid, pagesize) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const pageidFiler = Number(pageid) < 0 ? 0 : pageid; const pagesizeFiler = Number(pagesize) < 5 ? 5 : pagesize; const From = pageidFiler * pagesizeFiler; @@ -347,6 +366,8 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } export async function addPublish(Params) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + try{ Params._id = getNanoid(32) Params.name = filterString(Params.name) @@ -382,8 +403,9 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } } - export async function editPublish(Params) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + try{ Params._id = filterString(Params._id) Params.name = filterString(Params.name) @@ -409,6 +431,8 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } export async function deletePublish(Params) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + try{ Params._id = filterString(Params._id) Params.appId = filterString(Params.appId) @@ -427,6 +451,8 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } export async function getPublish(id, userId) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const idFilter = filterString(id) const userIdFilter = Number(userId) @@ -443,6 +469,8 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } export async function getChatLogByAppIdAndUserId(appId, userId, pageid, pagesize) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const appIdFiler = filterString(appId); const userIdFiler = filterString(userId); const pageidFiler = Number(pageid) < 0 ? 0 : pageid; @@ -467,6 +495,8 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } export async function deleteUserLogByAppId(appId, userId) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const UpdateChatLog = db.prepare("update chatlog set current = 0 where appId = ? and userId = ?"); UpdateChatLog.run(appId, userId); UpdateChatLog.finalize(); @@ -475,6 +505,8 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } export async function deleteUserLogByChatlogId(appId, userId, id) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const UpdateChatLog = db.prepare("delete from chatlog where appId = ? and userId = ? and _id = ?"); UpdateChatLog.run(appId, userId, id); UpdateChatLog.finalize(); diff --git a/express/src/utils/dataset.js b/express/src/utils/dataset.js index 4eb5131..20f07e4 100644 --- a/express/src/utils/dataset.js +++ b/express/src/utils/dataset.js @@ -1,16 +1,11 @@ -import * as fs from 'fs' -import multer from 'multer' -import path from 'path' -import * as crypto from 'crypto' -import validator from 'validator'; -import { promisify } from 'util'; -import { DataDir, CONDENSE_TEMPLATE_INIT, QA_TEMPLATE_INIT } from './const.js'; -import { db, getDbRecord, getDbRecordALL } from './db.js' -import { filterString, log, formatDate, getNanoid } from './utils.js' +import { initChatBookDb } from './db.js' +import { filterString, log, getNanoid } from './utils.js' export async function addDataset(Params) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + try{ Params._id = filterString(Params._id) Params.name = filterString(Params.name) @@ -34,6 +29,8 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } export async function editDataset(Params) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + try{ Params._id = filterString(Params._id) Params.teamId = filterString(Params.teamId) @@ -57,6 +54,8 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } export async function deleteDataset(Params) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + try{ Params._id = filterString(Params.datasetId) Params.userId = filterString(Params.userId) @@ -73,6 +72,8 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } export async function getDataset(id, userId) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const idFilter = filterString(id) const userIdFilter = Number(userId) @@ -89,6 +90,8 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } export async function getDatasetPage(pageid, pagesize, userId) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const pageidFiler = Number(pageid) < 0 ? 0 : pageid; const pagesizeFiler = Number(pagesize) < 5 ? 5 : pagesize; const From = pageidFiler * pagesizeFiler; @@ -111,6 +114,8 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } export async function getCollectionPageByDataset(datasetId, pageid, pagesize, userId) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const datasetIdFileter = filterString(datasetId) const pageidFiler = Number(pageid) < 0 ? 0 : pageid; const pagesizeFiler = Number(pagesize) < 5 ? 5 : pagesize; @@ -134,6 +139,8 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } export async function getCollectionAll(pageid, pagesize) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const pageidFiler = Number(pageid) < 0 ? 0 : pageid; const pagesizeFiler = Number(pagesize) < 5 ? 5 : pagesize; const From = pageidFiler * pagesizeFiler; @@ -159,6 +166,8 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } export async function addCollection(Params) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + try{ Params._id = getNanoid(32) Params.name = filterString(Params.name) @@ -193,6 +202,8 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } export async function editCollection(Params) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + try{ Params._id = filterString(Params._id) Params.name = filterString(Params.name) @@ -216,6 +227,8 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } export async function uploadCollection(Params) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + try{ const type = filterString(Params.type) const datasetId = filterString(Params.datasetId) @@ -268,8 +281,9 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } - export async function deleteCollection(Params) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + try{ Params._id = filterString(Params._id) Params.datasetId = filterString(Params.datasetId) @@ -287,6 +301,8 @@ import { filterString, log, formatDate, getNanoid } from './utils.js' } export async function getCollection(id, userId) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const idFilter = filterString(id) const userIdFilter = Number(userId) diff --git a/express/src/utils/db.js b/express/src/utils/db.js index 5629d06..c2174fc 100644 --- a/express/src/utils/db.js +++ b/express/src/utils/db.js @@ -1,24 +1,54 @@ // globals.ts +import fs from 'fs' import sqlite3 from 'sqlite3'; import { promisify } from 'util'; -import { DataDir } from './const.js'; import { enableDir } from './utils.js'; let initialized = false; let ChatBookSetting = null +let db = null +let getDbRecord = null +let getDbRecordALL = null -export async function initChatBookSetting(Data) { +export function initChatBookSetting(Data) { console.log("ChatBookSettingChatBookSetting", ChatBookSetting) ChatBookSetting = Data } -// @ts-ignore -export const db = new sqlite3.Database(DataDir + '/ChatBookSqlite3.db', { encoding: 'utf8' }); +export function initChatBookDb() { -export const getDbRecord = promisify(db.get.bind(db)); -export const getDbRecordALL = promisify(db.all.bind(db)); + const DataDir = ChatBookSetting && ChatBookSetting.NodeStorageDirectory ? ChatBookSetting.NodeStorageDirectory : "D:\\GitHub\\ChatBook\\express\\data"; + + if(!isDirectorySync(DataDir)) { + return {DataDir: null, db: null, getDbRecord: null, getDbRecordALL: null} + } + + if(db == null) { + db = new sqlite3.Database(DataDir + '/ChatBookSqlite3.db', { encoding: 'utf8' }); + } + + if(getDbRecord == null) { + getDbRecord = promisify(db.get.bind(db)); + } + + if(getDbRecordALL == null) { + getDbRecordALL = promisify(db.all.bind(db)); + } + + return {DataDir, db, getDbRecord, getDbRecordALL} +} + +export async function initChatBookDbExec() { + let exeStatus = 0 + if (!initialized) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + + if(!isDirectorySync(DataDir)) { + return {DataDir: null, db: null, getDbRecord: null, getDbRecordALL: null} + } + + console.log("dbdbdbdbdbdbdb", db, DataDir) -export async function initChatBookDb() { enableDir(DataDir); enableDir(DataDir + '/audio/'); enableDir(DataDir + '/avatarforapp/'); @@ -30,7 +60,7 @@ export async function initChatBookDb() { enableDir(DataDir + '/uploadfiles/'); enableDir(DataDir + '/parsedfiles/'); enableDir(DataDir + '/video/'); - + db.serialize(() => { db.run(` CREATE TABLE IF NOT EXISTS collection ( @@ -67,16 +97,6 @@ export async function initChatBookDb() { userId TEXT not null ); `); - db.run(` - CREATE TABLE IF NOT EXISTS knowledge ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT not null, - summary TEXT not null, - timestamp INTEGER not null default 0, - userId INTEGER not null, - UNIQUE(name, userId) - ); - `); db.run(` CREATE TABLE IF NOT EXISTS chatlog ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -228,31 +248,6 @@ export async function initChatBookDb() { createtime INTEGER not null default 0 ); `); - db.run(` - CREATE TABLE IF NOT EXISTS agents ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - title TEXT KEY not null, - description TEXT not null, - tags TEXT not null, - config TEXT not null, - avatar TEXT not null, - author TEXT not null, - createDate TEXT not null, - status INTEGER not null default 1, - model TEXT not null default 'Gemini', - type INTEGER not null default 1, - userId TEXT not null default 0 - ); - `); - db.run(` - CREATE TABLE IF NOT EXISTS useragents ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - userId INTEGER not null default 0, - appId INTEGER not null default 0, - createtime INTEGER not null default 0, - UNIQUE(userId, appId) - ); - `); db.run(` CREATE TABLE IF NOT EXISTS app ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -313,7 +308,6 @@ export async function initChatBookDb() { userId INTEGER not null default 0 ); `); - db.run(` CREATE TABLE IF NOT EXISTS dataset ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -335,12 +329,7 @@ export async function initChatBookDb() { `); }); -} -export async function initChatBookDbExec() { - let exeStatus = 0 - if (!initialized) { - await initChatBookDb(); initialized = true; exeStatus = 1 } @@ -348,3 +337,12 @@ export async function initChatBookDbExec() { console.log("initChatBookDbExec initialized", initialized) } +function isDirectorySync(path) { + try { + const stats = fs.statSync(path); + return stats.isDirectory(); + } catch (err) { + console.error('isDirectorySync Error checking if path is a directory:', err); + return false; + } + } \ No newline at end of file diff --git a/express/src/utils/lancedb.js b/express/src/utils/lancedb.js index 58be6b6..f817770 100644 --- a/express/src/utils/lancedb.js +++ b/express/src/utils/lancedb.js @@ -1,6 +1,4 @@ -import express from 'express'; -import crypto from 'crypto'; -import { OpenAI } from "openai"; + import { ChatOpenAI } from "@langchain/openai"; import { PromptTemplate } from "@langchain/core/prompts"; import { connect } from "vectordb"; @@ -11,8 +9,6 @@ import { BytesOutputParser, StringOutputParser } from '@langchain/core/output_pa import { DataDir } from './const.js'; -//import { db, getDbRecord, getDbRecordALL } from './db.js' - import dotenv from 'dotenv'; dotenv.config(); diff --git a/express/src/utils/llms.js b/express/src/utils/llms.js index 4b06e67..6b13437 100644 --- a/express/src/utils/llms.js +++ b/express/src/utils/llms.js @@ -2,7 +2,6 @@ import * as fs from 'fs' import path from 'path' import axios from 'axios'; -import os from "node:os"; import { OpenAI } from "openai"; import { ChatOpenAI, OpenAIEmbeddings } from "@langchain/openai"; @@ -18,12 +17,6 @@ import { ChatGoogleGenerativeAI } from "@langchain/google-genai"; import { HarmBlockThreshold, HarmCategory } from "@google/generative-ai"; //import { ChatBaiduWenxin } from "@langchain/community/chat_models/baiduwenxin"; -import { MemoryVectorStore } from "langchain/vectorstores/memory"; -import { createRetrieverTool } from "langchain/agents/toolkits"; -import { pull } from "langchain/hub"; -import { createOpenAIFunctionsAgent } from "langchain/agents"; -import { AgentExecutor } from "langchain/agents"; - import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter'; import { PDFLoader } from 'langchain/document_loaders/fs/pdf'; import { TextLoader } from 'langchain/document_loaders/fs/text'; @@ -47,7 +40,7 @@ import { RunnableSequence } from '@langchain/core/runnables'; import { BytesOutputParser, StringOutputParser } from '@langchain/core/output_parsers'; import { DataDir } from './const.js'; -import { db, getDbRecord, getDbRecordALL } from './db.js' +import { initChatBookDb } from './db.js' import { getLLMSSetting, log, isFile, formatDateString, enableDir, getNanoid, writeFile } from './utils.js' import { createEmbeddingsFromList, getWebsiteUrlContext, formatMessage, rephraseInput, retrieveContext } from './lancedb.js'; @@ -71,6 +64,8 @@ let ChatBaiduWenxinModel = null } export async function ChatApp(_id, res, userId, question, history, template, appId, publishId, allowChatLog, temperature, datasetId, DatasetPrompt) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const Records = await (getDbRecord)("SELECT * from app where _id = ?", [appId]); const AppDataText = Records ? Records.data : null; @@ -124,6 +119,8 @@ let ChatBaiduWenxinModel = null } export async function chatOpenAI(_id, res, userId, question, history, template, appId, publishId, allowChatLog, temperature) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + ChatBookOpenAIStreamResponse = '' const startTime = performance.now() if(OPENAI_API_KEY) { @@ -191,6 +188,8 @@ let ChatBaiduWenxinModel = null } export async function chatOpenAIDataset(_id, res, userId, question, history, template, appId, publishId, allowChatLog, temperature, datasetId, DatasetPrompt) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + ChatBookOpenAIStreamResponse = '' const startTime = performance.now() if(OPENAI_API_KEY) { @@ -276,6 +275,8 @@ let ChatBaiduWenxinModel = null } export async function chatDeepSeek(_id, res, userId, question, history, template, appId, publishId, allowChatLog, temperature) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const startTime = performance.now() const pastMessages = []; if (template && template !== '') { @@ -372,6 +373,8 @@ let ChatBaiduWenxinModel = null } export function makeChainOpenAI(retriever, CONDENSE_TEMPLATE, QA_TEMPLATE) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const condenseQuestionPrompt = ChatPromptTemplate.fromTemplate(CONDENSE_TEMPLATE); const answerPrompt = ChatPromptTemplate.fromTemplate(QA_TEMPLATE); @@ -445,6 +448,8 @@ let ChatBaiduWenxinModel = null } export async function chatChatGemini(_id, res, userId, question, history, template, appId, publishId, allowChatLog) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const datasetId = '' await initChatBookGeminiStream(res, datasetId) const pastMessages = [] @@ -488,6 +493,8 @@ let ChatBaiduWenxinModel = null } export async function chatChatGeminiMindMap(res, userId, question, history, template, appId) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const datasetId = '' await initChatBookGeminiStream(res, datasetId) const TextPrompts = template && template != '' ? template : "\n 要求生成一份PPT的大纲,以行业总结性报告的形式显现,生成15-20页左右,每一页3-6个要点,每一个要点字数在10-30之间,返回格式为Markdown,标题格式使用: **标题名称** 的形式表达." @@ -549,6 +556,8 @@ let ChatBaiduWenxinModel = null } export async function chatChatBaiduWenxin(res, userId, question, history, template, appId) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const datasetId = '' await initChatBookBaiduWenxinStream(res, datasetId); if(!ChatBaiduWenxinModel) { @@ -591,6 +600,8 @@ let ChatBaiduWenxinModel = null } export async function GenereateImageUsingDallE2(res, userId, question, size='1024x1024') { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const datasetId = '' getLLMSSettingData = await getLLMSSetting(datasetId); const OPENAI_API_BASE = getLLMSSettingData.OPENAI_API_BASE ?? "https://api.openai.com/v1"; @@ -658,6 +669,8 @@ let ChatBaiduWenxinModel = null } export async function GenereateAudioUsingTTS(res, ModelId, userId, question, voice='alloy', appId) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + getLLMSSettingData = await getLLMSSetting(ModelId); const OPENAI_API_BASE = getLLMSSettingData.OPENAI_API_BASE ?? "https://api.openai.com/v1"; const OPENAI_API_KEY = getLLMSSettingData.OPENAI_API_KEY; @@ -710,6 +723,8 @@ let ChatBaiduWenxinModel = null } export async function outputAudio(res, file) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + try { const fileList = file.split('_') const FileName = DataDir + "/audio/" + fileList[0] + "/" + file + ".mp3"; @@ -730,6 +745,8 @@ let ChatBaiduWenxinModel = null } export async function compressPng(file) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const FileName = path.join(DataDir, "/image/"+ file + ".png"); const FileNameNew = path.join(DataDir, "/image/"+ file + "_thumbnail.png"); if(!isFile(FileNameNew) && isFile(FileName)) { @@ -745,6 +762,8 @@ let ChatBaiduWenxinModel = null } export async function compressImageForImage(file, width, height) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const FileName = path.join(DataDir, "/imageforimage/"+ file); const FileNameNew = path.join(DataDir, "/imageforimage/Resize_" + (width ? width+'_'+file : height+'_'+file) ); if(!isFile(FileNameNew) && isFile(FileName)) { @@ -766,6 +785,8 @@ let ChatBaiduWenxinModel = null } export async function outputImage(res, file) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + try { await compressPng(file); const FileName = path.join(DataDir, "/image/"+ file + "_thumbnail.png"); @@ -787,6 +808,8 @@ let ChatBaiduWenxinModel = null } export async function outputAvatarForApp(res, file) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + try { await compressPng(file); const FileName = path.join(DataDir, "/avatarforapp/"+ file); @@ -808,6 +831,8 @@ let ChatBaiduWenxinModel = null } export async function outputAvatarForDataset(res, file) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + try { await compressPng(file); const FileName = path.join(DataDir, "/avatarfordataset/"+ file); @@ -829,6 +854,8 @@ let ChatBaiduWenxinModel = null } export async function outputImageOrigin(res, file) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + try { const FileName = path.join(DataDir, "/image/"+ file + ".png"); if(isFile(FileName)) { @@ -849,6 +876,8 @@ let ChatBaiduWenxinModel = null //此处只做数据转文本操作, 向量化数据在另外一个函数里面. export async function parseFilesAndWeb() { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + try { const RecordsAll = await getDbRecordALL(`SELECT * from collection where status = '0' order by id asc limit 10`); @@ -918,6 +947,8 @@ let ChatBaiduWenxinModel = null } export async function vectorDdProcess() { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + try { const RecordsAll = await getDbRecordALL(`SELECT * from dataset where syncStatus = '1' limit 1`); await Promise.all(RecordsAll.map(async (DatasetItem)=>{ diff --git a/express/src/utils/user.js b/express/src/utils/user.js index f406c87..c32cb18 100644 --- a/express/src/utils/user.js +++ b/express/src/utils/user.js @@ -8,8 +8,8 @@ dotenv.config(); - import { db, getDbRecord, getDbRecordALL } from './db.js' - + import { initChatBookDb } from './db.js' + const secretKey = process.env.JWT_TOKEN_SECRET_KEY || "ChatBookAI"; export const createJwtToken = (userId, email, role) => { @@ -62,6 +62,8 @@ } export async function checkUserTokenXWE(token) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const buffer = Buffer.from(token, 'base64'); const TxText = buffer.toString('utf-8'); const Tx = JSON.parse(TxText) @@ -126,6 +128,8 @@ } export async function userLoginLog(req, email, action, msg) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const agent = useragent.parse(req.headers['user-agent']); const BrowserType = agent.family const BrowserVersion = agent.toVersion() @@ -191,7 +195,9 @@ } } - export async function changeUserPasswordByToken(token, data) { + export async function changeUserPasswordByToken(token, data) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const checkUserTokenData = await checkUserToken(token); if(checkUserTokenData && checkUserTokenData.data && checkUserTokenData.data.email && checkUserTokenData.data.role == 'admin') { const getOneUserData = await getOneUser(checkUserTokenData.data.email); @@ -230,6 +236,8 @@ } export async function changeUserDetail(token, data) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const checkUserTokenData = await checkUserToken(token); if(checkUserTokenData && checkUserTokenData.data && checkUserTokenData.data.email && checkUserTokenData.data.role == 'admin') { const updateSetting = db.prepare('update user set firstname = ?, lastname = ?, organization = ?, mobile = ?, address = ?, state = ?, country = ?, language = ? where email = ?'); @@ -245,6 +253,8 @@ } export async function changeUserStatus(token, data) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const checkUserTokenData = await checkUserToken(token); if(checkUserTokenData && checkUserTokenData.data && checkUserTokenData.data.email && checkUserTokenData.data.role == 'admin') { const updateSetting = db.prepare('update user set user_status = ? where id = ?'); @@ -260,6 +270,8 @@ } export async function registerUser(email, username, password, confirm_password, language) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + try{ if(password != confirm_password) { @@ -286,6 +298,8 @@ } export async function addUser(token, data) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const checkUserTokenData = await checkUserToken(token); if(checkUserTokenData && checkUserTokenData.data && checkUserTokenData.data.email && checkUserTokenData.data.role == 'admin' && data && data.email && data.username) { @@ -312,6 +326,8 @@ } export async function editUser(token, data) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const checkUserTokenData = await checkUserToken(token); if(checkUserTokenData && checkUserTokenData.data && checkUserTokenData.data.email && checkUserTokenData.data.role == 'admin') { const updateSetting = db.prepare('update user set firstname = ?, lastname = ?, organization = ?, mobile = ?, address = ?, state = ?, country = ?, language = ? where email = ?'); @@ -328,6 +344,8 @@ } export async function deleteUser(token, data) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const checkUserTokenData = await checkUserToken(token); if(checkUserTokenData && checkUserTokenData.data && checkUserTokenData.data.email && checkUserTokenData.data.role == 'admin' && data && data.email && data.username) { const deleteSetting = db.prepare('delete from user where email = ? and username = ?'); @@ -344,12 +362,16 @@ } export async function getOneUser(email) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const Records = await (getDbRecord)("SELECT * from user where email = ? ", [email]); return Records ? Records : null; } export async function getOneUserByToken(token) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const checkUserTokenData = await checkUserToken(token); if(checkUserTokenData && checkUserTokenData.data && checkUserTokenData.data.email) { const getOneUserData = await getOneUser(checkUserTokenData.data.email); @@ -381,6 +403,8 @@ } export async function getUsers(pageid, pagesize, data) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const pageidFiler = Number(pageid) < 0 ? 0 : pageid; const pagesizeFiler = Number(pagesize) < 5 ? 5 : pagesize; const From = pageidFiler * pagesizeFiler; @@ -425,6 +449,8 @@ } export async function getUserLogs(email, pageid, pagesize) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const pageidFiler = Number(pageid) < 0 ? 0 : pageid; const pagesizeFiler = Number(pagesize) < 5 ? 5 : pagesize; const From = pageidFiler * pagesizeFiler; @@ -448,6 +474,8 @@ } export async function getUserLogsAll(pageid, pagesize) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const pageidFiler = Number(pageid) < 0 ? 0 : pageid; const pagesizeFiler = Number(pagesize) < 5 ? 5 : pagesize; const From = pageidFiler * pagesizeFiler; @@ -480,6 +508,8 @@ } export async function updateUserImageFavorite(token, data) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + console.log("data", data) const checkUserTokenData = await checkUserToken(token); if(checkUserTokenData && checkUserTokenData.data && checkUserTokenData.data.id && (checkUserTokenData.data.role == 'admin' || checkUserTokenData.data.role == 'user' && data && data.id && data.status != null)) { @@ -504,6 +534,8 @@ } export async function updateUserVideoFavorite(token, data) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + console.log("data", data) const checkUserTokenData = await checkUserToken(token); if(checkUserTokenData && checkUserTokenData.data && checkUserTokenData.data.id && (checkUserTokenData.data.role == 'admin' || checkUserTokenData.data.role == 'user' && data && data.id && data.status != null)) { @@ -526,85 +558,3 @@ return {"status":"error", "msg":"Token is invalid in changeUserDetail"} } } - - - export async function addUserAgent(token, data) { - console.log("data", data) - const checkUserTokenData = await checkUserToken(token); - if(checkUserTokenData && checkUserTokenData.data && checkUserTokenData.data.id && (checkUserTokenData.data.role == 'admin' || checkUserTokenData.data.role == 'user' && data && data.appId)) { - console.log("addUserAgent", checkUserTokenData) - const insertUser = db.prepare('INSERT OR REPLACE INTO useragents (userId, appId, createtime) VALUES (?, ?, ?)'); - insertUser.run(checkUserTokenData.data.id, data.appId, Date.now()); - insertUser.finalize(); - - return {"status":"ok", "msg":"Add agent successful"} - } - else { - return {"status":"error", "msg":"Token is invalid in addUserAgent"} - } - } - - export async function deleteUserAgent(token, data) { - console.log("data", data) - const checkUserTokenData = await checkUserToken(token); - if(checkUserTokenData && checkUserTokenData.data && checkUserTokenData.data.id && (checkUserTokenData.data.role == 'admin' || checkUserTokenData.data.role == 'user' && data && data.appId)) { - console.log("addUserAgent", checkUserTokenData) - const commitSql = db.prepare('delete from useragents where userId = ? and appId = ?'); - commitSql.run(checkUserTokenData.data.id, data.appId); - commitSql.finalize(); - return {"status":"ok", "msg":"Cancel agent successful"} - } - else { - return {"status":"error", "msg":"Token is invalid in addUserAgent"} - } - } - - export async function getUserAgents(userId, pageid, pagesize) { - const pageidFiler = Number(pageid) < 0 ? 0 : pageid; - const pagesizeFiler = Number(pagesize) < 5 ? 5 : pagesize; - const From = pageidFiler * pagesizeFiler; - console.log("pageidFiler", pageidFiler) - console.log("pagesizeFiler", pagesizeFiler) - - const Records = await (getDbRecord)("SELECT COUNT(*) AS NUM from useragents where userId = ? ", [userId]); - const RecordsTotal = Records ? Records.NUM : 0; - - const RecordsAll = await (getDbRecordALL)(`SELECT * FROM useragents WHERE userId = ? ORDER BY id DESC LIMIT ? OFFSET ? `, [userId, pagesizeFiler, From]) || []; - - const RS = {}; - RS['allpages'] = Math.ceil(RecordsTotal/pagesizeFiler); - RS['data'] = RecordsAll.filter(element => element !== null && element !== undefined && element !== ''); - RS['from'] = From; - RS['pageid'] = pageidFiler; - RS['pagesize'] = pagesizeFiler; - RS['total'] = RecordsTotal; - - return RS; - } - - export async function getMyAgents(userId, pageid, pagesize) { - const pageidFiler = Number(pageid) < 0 ? 0 : pageid; - const pagesizeFiler = Number(pagesize) < 5 ? 5 : pagesize; - const From = pageidFiler * pagesizeFiler; - console.log("pageidFiler", pageidFiler) - console.log("pagesizeFiler", pagesizeFiler) - - const Records = await (getDbRecord)("SELECT COUNT(*) AS NUM from useragents where userId = ? ", [userId]); - const RecordsTotal = Records ? Records.NUM : 0; - - const RecordsAll = await (getDbRecordALL)(`SELECT * FROM useragents WHERE userId = ? ORDER BY id DESC LIMIT ? OFFSET ? `, [userId, pagesizeFiler, From]) || []; - const AgentsList = RecordsAll.map(element => element.appId); - const PlaceHolders = AgentsList.map(() => '?').join(','); - const RecordsList = await (getDbRecordALL)(`SELECT * FROM agents WHERE id IN (${PlaceHolders}) ORDER BY id DESC`, AgentsList) || []; - - const RS = {}; - RS['allpages'] = Math.ceil(RecordsTotal/pagesizeFiler); - RS['data'] = RecordsList; - RS['RecordsAll'] = RecordsAll; - RS['from'] = From; - RS['pageid'] = pageidFiler; - RS['pagesize'] = pagesizeFiler; - RS['total'] = RecordsTotal; - - return RS; - } \ No newline at end of file diff --git a/express/src/utils/utils.js b/express/src/utils/utils.js index f083995..c4f7378 100644 --- a/express/src/utils/utils.js +++ b/express/src/utils/utils.js @@ -3,12 +3,11 @@ import multer from 'multer' import path from 'path' import * as crypto from 'crypto' import validator from 'validator'; -import { promisify } from 'util'; -import { DataDir, CONDENSE_TEMPLATE_INIT, QA_TEMPLATE_INIT } from './const.js'; import { randomBytes } from 'crypto'; -import { db, getDbRecord, getDbRecordALL } from './db.js' +import { initChatBookDb } from './db.js' import iconv from 'iconv-lite'; + export function enableDir(directoryPath) { try { fs.accessSync(directoryPath, fs.constants.F_OK); @@ -25,6 +24,8 @@ export function enableDir(directoryPath) { } export async function getLLMSSetting(datasetId) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const datasetIdFilter = filterString(datasetId) const RecordsAll = await (getDbRecordALL)(`SELECT name,content from setting where type='openaisetting' and datasetId = ? `, [datasetIdFilter]); @@ -39,6 +40,8 @@ export async function getLLMSSetting(datasetId) { } export async function setOpenAISetting(Params) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const datasetIdFilter = filterString(Params.datasetId) const userIdFilter = filterString(Params.userId) try { @@ -58,6 +61,8 @@ export async function setOpenAISetting(Params) { } export async function getTemplate(datasetId, userId) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const datasetIdFilter = filterString(datasetId) const userIdFilter = Number(userId) @@ -74,6 +79,8 @@ export async function getTemplate(datasetId, userId) { } export async function setTemplate(Params) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + try{ const datasetIdFilter = Number(Params.datasetId) const userIdFilter = Params.userId @@ -90,67 +97,6 @@ export async function setTemplate(Params) { return {"status":"ok", "msg":"Update Success"} } -export async function addKnowledge(Params) { - try{ - console.log("ParamsParamsParamsParamsParams", Params) - const userIdFilter = Params.userId - Params.name = filterString(Params.name) - Params.summary = filterString(Params.summary) - - const Records = await (getDbRecord)("SELECT id from knowledge where name = ? and userId = ?", [Params.name, userIdFilter]); - const RecordId = Records ? Records.id : 0; - - console.log("RecordId", RecordId, Params.userId) - if(RecordId > 0) { - Params.id = RecordId - setKnowledge(Params) - } - else { - const insertSetting = db.prepare('INSERT OR REPLACE INTO knowledge (name, summary, timestamp, userId) VALUES (?, ?, ?, ?)'); - insertSetting.run(Params.name, Params.summary, Date.now(), userIdFilter); - - insertSetting.run(Params.name, Params.summary, Date.now(), userIdFilter, function(err) { - if (err) { - console.error(err.message); - - return; - } - - //@ts-ignore - const lastInsertId = this.lastID; - console.log('Last inserted ID:', lastInsertId); - insertSetting.finalize(); - if(lastInsertId) { - const TemplateInfo = {datasetId: lastInsertId, CONDENSE_TEMPLATE: CONDENSE_TEMPLATE_INIT, QA_TEMPLATE: QA_TEMPLATE_INIT, userId: userIdFilter} - setTemplate(TemplateInfo) - } - - }); - } - } - catch (error) { - console.log('Error setOpenAISetting:', error.message); - } - - return {"status":"ok", "msg":"Update Success"} -} - -export async function setKnowledge(Params) { - try{ - Params.id = Number(Params.id) - Params.name = filterString(Params.name) - Params.summary = filterString(Params.summary) - const updateSetting = db.prepare('update knowledge set name = ?, summary = ?, timestamp = ? where id = ?'); - updateSetting.run(Params.name, Params.summary, Date.now(), Params.id); - updateSetting.finalize(); - } - catch (error) { - console.log('Error setOpenAISetting:', error.message); - } - - return {"status":"ok", "msg":"Update Success"} -} - export function uploadfiles() { const storage = multer.diskStorage({ destination: (req, file, cb) => { @@ -257,6 +203,8 @@ export async function uploadfilesInsertIntoDb(files, datasetId, userId) { } export async function getFilesPage(pageid, pagesize) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const pageidFiler = Number(pageid) < 0 ? 0 : pageid; const pagesizeFiler = Number(pagesize) < 5 ? 5 : pagesize; const From = pageidFiler * pagesizeFiler; @@ -296,15 +244,17 @@ export async function getFilesPage(pageid, pagesize) { } export async function getFilesDatasetId(datasetId, pageid, pagesize) { - const KnowledgeIdFiler = Number(datasetId) < 0 ? 0 : datasetId; + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + + const datasetIdFiler = Number(datasetId) < 0 ? 0 : datasetId; const pageidFiler = Number(pageid) < 0 ? 0 : pageid; const pagesizeFiler = Number(pagesize) < 5 ? 5 : pagesize; const From = pageidFiler * pagesizeFiler; - const Records = await (getDbRecord)("SELECT COUNT(*) AS NUM from files where datasetId = ?", [KnowledgeIdFiler]); + const Records = await (getDbRecord)("SELECT COUNT(*) AS NUM from files where datasetId = ?", [datasetIdFiler]); const RecordsTotal = Records ? Records.NUM : 0; - const RecordsAll = await (getDbRecordALL)(`SELECT * from files where datasetId = ? order by status desc, timestamp desc limit ? OFFSET ? `, [KnowledgeIdFiler, pagesizeFiler, From]); + const RecordsAll = await (getDbRecordALL)(`SELECT * from files where datasetId = ? order by status desc, timestamp desc limit ? OFFSET ? `, [datasetIdFiler, pagesizeFiler, From]); let RSDATA = [] if(RecordsAll != undefined) { @@ -336,23 +286,27 @@ export async function getFilesDatasetId(datasetId, pageid, pagesize) { } export async function getFilesNotParsed() { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const RecordsAll = await (getDbRecordALL)(`SELECT * from files where status = '0' order by timestamp asc limit 10 `); return RecordsAll; } -export async function getChatLogByKnowledgeIdAndUserId(datasetId, userId, pageid, pagesize) { - const KnowledgeIdFiler = filterString(datasetId); +export async function getChatLogByDatasetIdAndUserId(datasetId, userId, pageid, pagesize) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + + const datasetIdFiler = filterString(datasetId); const userIdFiler = Number(userId) < 0 ? 0 : userId; const pageidFiler = Number(pageid) < 0 ? 0 : pageid; const pagesizeFiler = Number(pagesize) < 5 ? 5 : pagesize; const From = pageidFiler * pagesizeFiler; - const Records = await (getDbRecord)("SELECT COUNT(*) AS NUM from chatlog where current = 1 and appId = ? and userId = ?", [KnowledgeIdFiler, userIdFiler]); + const Records = await (getDbRecord)("SELECT COUNT(*) AS NUM from chatlog where current = 1 and appId = ? and userId = ?", [datasetIdFiler, userIdFiler]); const RecordsTotal = Records ? Records.NUM : 0; - const RecordsAll = await (getDbRecordALL)(`SELECT * from chatlog where current = 1 and appId = ? and userId = ? order by id desc limit ? OFFSET ? `, [KnowledgeIdFiler, userIdFiler, pagesizeFiler, From]); + const RecordsAll = await (getDbRecordALL)(`SELECT * from chatlog where current = 1 and appId = ? and userId = ? order by id desc limit ? OFFSET ? `, [datasetIdFiler, userIdFiler, pagesizeFiler, From]); const RS = {}; RS['allpages'] = Math.ceil(RecordsTotal/pagesizeFiler); @@ -366,6 +320,8 @@ export async function getChatLogByKnowledgeIdAndUserId(datasetId, userId, pageid } export async function getLogsPage(pageid, pagesize) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const pageidFiler = Number(pageid) < 0 ? 0 : pageid; const pagesizeFiler = Number(pagesize) < 5 ? 5 : pagesize; const From = pageidFiler * pagesizeFiler; @@ -386,29 +342,6 @@ export async function getLogsPage(pageid, pagesize) { return RS; } -export async function getKnowledgePage(pageid, pagesize) { - const pageidFiler = Number(pageid) < 0 ? 0 : pageid; - const pagesizeFiler = Number(pagesize) < 5 ? 5 : pagesize; - const From = pageidFiler * pagesizeFiler; - console.log("pageidFiler", pageidFiler) - console.log("pagesizeFiler", pagesizeFiler) - - - const Records = await (getDbRecord)("SELECT COUNT(*) AS NUM from knowledge"); - const RecordsTotal = Records ? Records.NUM : 0; - const RecordsAll = await (getDbRecordALL)(`SELECT * from knowledge order by id desc limit ? OFFSET ? `, [pagesizeFiler, From]); - - const RS = {}; - RS['allpages'] = Math.ceil(RecordsTotal/pagesizeFiler); - RS['data'] = RecordsAll.filter(element => element !== null && element !== undefined && element !== ''); - RS['from'] = From; - RS['pageid'] = pageidFiler; - RS['pagesize'] = pagesizeFiler; - RS['total'] = RecordsTotal; - - return RS; -} - export function timestampToDate(timestamp) { const date = new Date(Number(timestamp) * 1000); const year = date.getFullYear(); @@ -419,6 +352,8 @@ export function timestampToDate(timestamp) { } export async function log(appId, publishId, userId, Action1, Action2, Action3, Action4, Action5, Action6, Action7, Action8, Action9, Action10) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const currentDate = new Date(); const currentDateTime = currentDate.toLocaleString(); const content = JSON.stringify(Action1) +" "+ JSON.stringify(Action2) +" "+ JSON.stringify(Action3) +" "+ JSON.stringify(Action4) +" "+ JSON.stringify(Action5) +" "+ JSON.stringify(Action6) +" "+ JSON.stringify(Action7) +" "+ JSON.stringify(Action8) +" "+ JSON.stringify(Action9) +" "+ JSON.stringify(Action10); @@ -429,6 +364,8 @@ export async function log(appId, publishId, userId, Action1, Action2, Action3, A } export async function deleteLog() { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const Records = await (getDbRecord)("SELECT MAX(id) AS NUM FROM logs"); const MaxId = Records ? Records.NUM : 0; if(MaxId > 1000) { @@ -440,6 +377,8 @@ export async function deleteLog() { } export async function clearShortLogs() { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const DeleteLog = db.prepare("delete from logs where length(content) < 50"); DeleteLog.run(); DeleteLog.finalize(); @@ -447,13 +386,17 @@ export async function clearShortLogs() { } export async function clearAllLogs() { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const DeleteLog = db.prepare("delete from logs"); DeleteLog.run(); DeleteLog.finalize(); return {"status":"ok", "msg":"Deleted Success"} } -export async function deleteUserLogByKnowledgeId(datasetId, userId) { +export async function deleteUserLogByDatasetId(datasetId, userId) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const UpdateChatLog = db.prepare("update chatlog set current = 0 where appId = ? and userId = ?"); UpdateChatLog.run(datasetId, userId); UpdateChatLog.finalize(); @@ -599,6 +542,8 @@ export const isEmailValid = (email) => { }; export async function wholeSiteStatics() { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const NewUserPerDayData = await (getDbRecordALL)(`SELECT strftime('%Y-%m-%d', datetime(createtime / 1000, 'unixepoch')) AS date, count(*) AS NUM from user group by date order by date asc`); const NewUserPerDay = NewUserPerDayData.map(Item => Item.NUM) @@ -626,13 +571,12 @@ export async function wholeSiteStatics() { const Records4 = await (getDbRecord)("SELECT COUNT(*) AS NUM from files"); const TotalFiles = Records4 ? Records4.NUM : 0; - const Records5 = await (getDbRecord)("SELECT COUNT(*) AS NUM from knowledge"); - const TotalKnowledges = Records5 ? Records5.NUM : 0; - - return {NewUserPerDay, NewImagesPerDay, NewFilesPerDay, NewActivitesPerDay, DateList, TotalImages, TotalActivites, TotalUsers, TotalFiles, TotalKnowledges} + return {NewUserPerDay, NewImagesPerDay, NewFilesPerDay, NewActivitesPerDay, DateList, TotalImages, TotalActivites, TotalUsers, TotalFiles} } export async function getAllImages(userId, pageid, pagesize) { + const { DataDir, db, getDbRecord, getDbRecordALL } = initChatBookDb() + const pageidFiler = Number(pageid) < 0 ? 0 : pageid; const pagesizeFiler = Number(pagesize) < 5 ? 5 : pagesize; const From = pageidFiler * pagesizeFiler; @@ -683,121 +627,6 @@ export function removeDuplicates(words) { return uniqueWords.join(','); } -export async function getAgentsPage(pageid, pagesize) { - const pageidFiler = Number(pageid) < 0 ? 0 : pageid; - const pagesizeFiler = Number(pagesize) < 5 ? 5 : pagesize; - const From = pageidFiler * pagesizeFiler; - console.log("pageidFiler", pageidFiler) - console.log("pagesizeFiler", pagesizeFiler) - - - const Records = await (getDbRecord)("SELECT COUNT(*) AS NUM from agents"); - const RecordsTotal = Records ? Records.NUM : 0; - const RecordsAll = await (getDbRecordALL)(`SELECT * from agents order by id desc limit ? OFFSET ? `, [pagesizeFiler, From]); - - const RS = {}; - RS['allpages'] = Math.ceil(RecordsTotal/pagesizeFiler); - RS['data'] = RecordsAll.filter(element => element !== null && element !== undefined && element !== ''); - RS['from'] = From; - RS['pageid'] = pageidFiler; - RS['pagesize'] = pagesizeFiler; - RS['total'] = RecordsTotal; - - return RS; -} - -export async function getAgentsEnabledList(pageid, pagesize, type, search) { - const pageidFiler = Number(pageid) < 0 ? 0 : pageid; - const pagesizeFiler = Number(pagesize) < 5 ? 5 : pagesize; - const From = pageidFiler * pagesizeFiler; - console.log("pageidFiler", pageidFiler) - console.log("pagesizeFiler", pagesizeFiler) - console.log("type", type) - console.log("search", search) - - - let Records = null; - let RecordsTotal = Records ? Records.NUM : 0; - let RecordsAll = []; - if(type != "ALL" && type != "全部") { - Records = await (getDbRecord)("SELECT COUNT(*) AS NUM from agents where status = 1 and tags like ?", [`%${type}%`]); - RecordsTotal = Records ? Records.NUM : 0; - RecordsAll = await (getDbRecordALL)(`SELECT * from agents where status = 1 and tags like ? order by id desc limit ? OFFSET ? `, [`%${type}%`, pagesizeFiler, From]); - console.log("RecordsAll", RecordsAll, type, search) - } - else if(search != "ALL" && search != "全部") { - Records = await (getDbRecord)("SELECT COUNT(*) AS NUM from agents where status = 1 and title like ?", [`%${search}%`]); - RecordsTotal = Records ? Records.NUM : 0; - RecordsAll = await (getDbRecordALL)(`SELECT * from agents where status = 1 and title like ? order by id desc limit ? OFFSET ? `, [`%${search}%`, pagesizeFiler, From]); - } - else { - Records = await (getDbRecord)("SELECT COUNT(*) AS NUM from agents where status = 1"); - RecordsTotal = Records ? Records.NUM : 0; - RecordsAll = await (getDbRecordALL)(`SELECT * from agents where status = 1 order by id desc limit ? OFFSET ? `, [pagesizeFiler, From]); - } - - const RS = {}; - RS['allpages'] = Math.ceil(RecordsTotal/pagesizeFiler); - RS['data'] = RecordsAll.filter(element => element !== null && element !== undefined && element !== ''); - RS['from'] = From; - RS['pageid'] = pageidFiler; - RS['pagesize'] = pagesizeFiler; - RS['total'] = RecordsTotal; - - return RS; -} - -export async function addAgent(Params) { - try{ - console.log("ParamsParamsParamsParamsParams", Params) - Params.title = filterString(Params.title) - Params.description = filterString(Params.description) - Params.tags = filterString(Params.tags) - Params.config = filterString(Params.config) - Params.avatar = filterString(Params.avatar) - Params.author = filterString(Params.author) - Params.model = filterString(Params.model) - - const Records = await (getDbRecord)("SELECT id from agents where title = ?", [Params.title]); - const RecordId = Records ? Records.id : 0; - - console.log("RecordId", RecordId, Params.userId) - if(RecordId > 0) { - Params.id = RecordId - editAgent(Params) - return {"status":"ok", "msg":"Update Success"} - } - else { - const formatDateValue = formatDate(Date.now()) - const insertSetting = db.prepare('INSERT OR IGNORE INTO agents (title, description, tags, config, avatar, author, createDate, status, model, type, userId) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'); - insertSetting.run(Params.title, Params.description, Params.tags, Params.config, Params.avatar, Params.author, formatDateValue, 1, Params.model, 1, Params.userId); - insertSetting.finalize(); - return {"status":"ok", "msg":"Add Success"} - } - } - catch (error) { - console.log('Error setOpenAISetting:', error.message); - return {"status":"error", "msg":error.message} - } -} - -export async function editAgent(Params) { - try{ - Params.id = Number(Params.id) - Params.name = filterString(Params.name) - Params.summary = filterString(Params.summary) - const formatDateValue = formatDate(Date.now()) - const updateSetting = db.prepare('update agents set title = ?, description = ?, tags = ?, config = ?, avatar = ?, author = ?, createDate = ?, status = ?, model = ?, type = ? where id = ?'); - updateSetting.run(Params.title, Params.description, Params.tags, Params.config, Params.avatar, Params.author, formatDateValue, Params.status, Params.model, Params.type, Params.id); - updateSetting.finalize(); - } - catch (error) { - console.log('Error setOpenAISetting:', error.message); - } - - return {"status":"ok", "msg":"Update Success"} -} - export const getNanoid = (size = 12) => { const bytes = randomBytes(16); const hexString = bytes.toString('hex');