Skip to content

Commit

Permalink
ChatBookDbInit
Browse files Browse the repository at this point in the history
  • Loading branch information
@SmartSchoolAI
SmartSchoolAI committed Jun 2, 2024
1 parent d2b5727 commit 57e0b89
Show file tree
Hide file tree
Showing 4 changed files with 16 additions and 134 deletions.
7 changes: 1 addition & 6 deletions express/src/app.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import cron from 'node-cron';
import dotenv from 'dotenv';
import { dirname, join } from 'path';

import { ChatBookDbInit, initChatBookSetting } from './utils/db.js';
import { ChatBookDbInit, isElectron } from './utils/db.js';
import { parseFilesAndWeb, vectorDdProcess } from './utils/llms.js';

import userRouter from './router/user.js'
Expand All @@ -31,13 +31,8 @@ dotenv.config();

//Initial Database and Folder

function isElectron() {
return typeof process !== 'undefined' && !!(process.versions).electron;
}

//Running in express, not in electron
if(isElectron() == false) {
initChatBookSetting({NodeStorageDirectory: './data'})
ChatBookDbInit()
}

Expand Down
29 changes: 1 addition & 28 deletions express/src/router/utils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

import { MenuListAdmin, MenuListUser } from '../utils/const.js';
import { checkUserToken } from '../utils/user.js';
import { getLogsPage, clearShortLogs, clearAllLogs, getTemplate, getLLMSSetting, getFilesPage, getFilesDatasetId, getChatLogByDatasetIdAndUserId, setOpenAISetting, setTemplate, wholeSiteStatics, getAllImages, deleteUserLogByDatasetId } from '../utils/utils.js';
import { getLogsPage, clearShortLogs, clearAllLogs, getTemplate, getLLMSSetting, getChatLogByDatasetIdAndUserId, setOpenAISetting, setTemplate, wholeSiteStatics, getAllImages, deleteUserLogByDatasetId } from '../utils/utils.js';

const app = express();

Expand Down Expand Up @@ -62,33 +62,6 @@
res.end();
});

app.get('/api/files/:pageid/:pagesize', async (req, res) => {
const { pageid, pagesize } = req.params;
const { authorization } = req.headers;
const checkUserTokenData = await checkUserToken(authorization);
if(checkUserTokenData && checkUserTokenData.data && checkUserTokenData.data.email && checkUserTokenData.data.role == 'admin') {
const getFilesPageData = await getFilesPage(Number(pageid), Number(pagesize));
res.status(200).json(getFilesPageData);
}
else {
res.status(200).json({"status":"error", "msg":"Token is invalid", "data": null});
}
res.end();
});

app.get('/api/files/:datasetId/:pageid/:pagesize', async (req, res) => {
const { datasetId, pageid, pagesize } = req.params;
const { authorization } = req.headers;
const checkUserTokenData = await checkUserToken(authorization);
if(checkUserTokenData && checkUserTokenData.data && checkUserTokenData.data.email && checkUserTokenData.data.role == 'admin') {
const getFilesPageData = await getFilesDatasetId(datasetId, Number(pageid), Number(pagesize));
res.status(200).json(getFilesPageData);
}
else {
res.status(200).json({"status":"error", "msg":"Token is invalid", "data": null});
}
res.end();
});

app.get('/api/getopenai/:id', async (req, res) => {
const { id } = req.params;
Expand Down
19 changes: 14 additions & 5 deletions express/src/utils/db.js
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

95 changes: 0 additions & 95 deletions express/src/utils/utils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,98 +272,6 @@ export async function uploadfilesInsertIntoDb(files, datasetId, userId) {

}

export async function getFilesPage(pageid, pagesize) {
const { DataDir, db, getDbRecord, getDbRecordALL } = ChatBookDbPool()

const pageidFiler = Number(pageid) < 0 ? 0 : pageid;
const pagesizeFiler = Number(pagesize) < 5 ? 5 : pagesize;
const From = pageidFiler * pagesizeFiler;

const Records = await (getDbRecord)("SELECT COUNT(*) AS NUM from files");
const RecordsTotal = Records ? Records.NUM : 0;
const RecordsAll = await (getDbRecordALL)(`SELECT * from files where 1=1 order by status desc, timestamp desc limit ? OFFSET ? `, [pagesizeFiler, From]);
let RSDATA = []
if(RecordsAll != undefined) {
RSDATA = await Promise.all(
RecordsAll.map(async (Item)=>{
let ItemStatus = "Ready To Parse"
switch(Item.status) {
case 1:
ItemStatus = 'Finished'
break;
case -1:
ItemStatus = 'File Not Exist'
break;
}

return {...Item, status:ItemStatus, timestamp: formatDateFromTimestamp(Item.timestamp)}
})
);

//log("getFilesPage", RSDATA)
}
const RS = {};
RS['allpages'] = Math.ceil(RecordsTotal/pagesizeFiler);
RS['data'] = RSDATA.filter(element => element !== null && element !== undefined && element !== '');
RS['from'] = From;
RS['pageid'] = pageidFiler;
RS['pagesize'] = pagesizeFiler;
RS['total'] = RecordsTotal;

return RS;
}

export async function getFilesDatasetId(datasetId, pageid, pagesize) {
const { DataDir, db, getDbRecord, getDbRecordALL } = ChatBookDbPool()

const datasetIdFiler = Number(datasetId) < 0 ? 0 : datasetId;
const pageidFiler = Number(pageid) < 0 ? 0 : pageid;
const pagesizeFiler = Number(pagesize) < 5 ? 5 : pagesize;
const From = pageidFiler * pagesizeFiler;


const Records = await (getDbRecord)("SELECT COUNT(*) AS NUM from files where datasetId = ?", [datasetIdFiler]);
const RecordsTotal = Records ? Records.NUM : 0;
const RecordsAll = await (getDbRecordALL)(`SELECT * from files where datasetId = ? order by status desc, timestamp desc limit ? OFFSET ? `, [datasetIdFiler, pagesizeFiler, From]);

let RSDATA = []
if(RecordsAll != undefined) {
RSDATA = await Promise.all(
RecordsAll.map(async (Item)=>{
let ItemStatus = "Ready To Parse"
switch(Item.status) {
case 1:
ItemStatus = 'Finished'
break;
case -1:
ItemStatus = 'File Not Exist'
break;
}

return {...Item, status:ItemStatus, timestamp: formatDateFromTimestamp(Item.timestamp)}
})
);
}
const RS = {};
RS['allpages'] = Math.ceil(RecordsTotal/pagesizeFiler);
RS['data'] = RSDATA.filter(element => element !== null && element !== undefined && element !== '');
RS['from'] = From;
RS['pageid'] = pageidFiler;
RS['pagesize'] = pagesizeFiler;
RS['total'] = RecordsTotal;

return RS;
}

export async function getFilesNotParsed() {
const { DataDir, db, getDbRecord, getDbRecordALL } = ChatBookDbPool()


const RecordsAll = await (getDbRecordALL)(`SELECT * from files where status = '0' order by timestamp asc limit 10 `);

return RecordsAll;
}

export async function getChatLogByDatasetIdAndUserId(datasetId, userId, pageid, pagesize) {
const { DataDir, db, getDbRecord, getDbRecordALL } = ChatBookDbPool()

Expand Down Expand Up @@ -622,9 +530,6 @@ export async function wholeSiteStatics() {
const NewImagesPerDayData = await (getDbRecordALL)(`SELECT strftime('%Y-%m-%d', datetime(createtime / 1000, 'unixepoch')) AS date, count(*) AS NUM from userimages group by date order by date asc`);
const NewImagesPerDay = NewImagesPerDayData.map(Item => Item.NUM)

const NewFilesPerDayData = await (getDbRecordALL)(`SELECT strftime('%Y-%m-%d', datetime(timestamp / 1000, 'unixepoch')) AS date, count(*) AS NUM from files group by date order by date asc`);
const NewFilesPerDay = NewFilesPerDayData.map(Item => Item.NUM)

const NewActivitesPerDayData = await (getDbRecordALL)(`SELECT strftime('%Y-%m-%d', datetime(timestamp / 1000, 'unixepoch')) AS date, count(*) AS NUM from chatlog group by date order by date asc`);
const NewActivitesPerDay = NewActivitesPerDayData.map(Item => Item.NUM)

Expand Down

0 comments on commit 57e0b89

Please sign in to comment.