Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Playground improvements #382

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Playground improvements #382

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bcffe15
Playground: Only fetch 16 documents instead of everything
cmorlok Feb 24, 2025
1769993
Playground: Same behaviour for tags and corresponds like normal proce…
cmorlok Feb 24, 2025
edba04e
Playground: Show all fields that are returned from analysis
cmorlok Feb 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 70 additions & 6 deletions public/js/playground-analyzer.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -602,7 +602,9 @@ class PlaygroundAnalyzer {
}));

const existingCorrespondent = docCard.querySelector('[data-correspondent]')?.dataset.correspondent;
const existingDocumentType = docCard.querySelector('[data-document_type]')?.dataset.document_type;
const existingTitle = docCard.querySelector('h3').textContent;
const existingDocumentDate = docCard.querySelector('[data-document_date]')?.dataset.document_date;

// Analyse durchführen
const analysisResponse = await fetch('/manual/playground', {
Expand All @@ -612,8 +614,6 @@ class PlaygroundAnalyzer {
},
body: JSON.stringify({
content: contentData.content,
existingTags: existingTags.map(t => t.id),
correspondent: existingCorrespondent,
prompt: prompt,
documentId: docId
})
Expand All @@ -630,7 +630,9 @@ class PlaygroundAnalyzer {
await this.updateDocumentCard(docCard, result.document, {
existingTags,
existingCorrespondent,
existingTitle
existingDocumentType,
existingTitle,
existingDocumentDate
});

} catch (error) {
Expand Down Expand Up @@ -659,7 +661,31 @@ class PlaygroundAnalyzer {
});
}
}


// Update document date
let documentDate = new Date(analysisResult.document_date);
if (!isNaN(documentDate) && analysisResult.document_date.valueOf() !== existing.existingDocumentDate) {
const dateElem = docCard.querySelector('[data-document_date]');
if (dateElem) {
const oldDocumentDate = dateElem.textContent.trim();
const infoContainer = document.createElement('div');
infoContainer.className = 'info-item';

const newValue = document.createElement('span');
newValue.className = 'updated-text text-sm truncate';
newValue.textContent = documentDate.toLocaleDateString();

const oldValue = document.createElement('span');
oldValue.className = 'old-value';
oldValue.textContent = oldDocumentDate;

infoContainer.appendChild(newValue);
infoContainer.appendChild(oldValue);

dateElem.parentNode.replaceChild(infoContainer, dateElem);
}
}

// Correspondent aktualisieren
if (analysisResult.correspondent && analysisResult.correspondent !== existing.existingCorrespondent) {
const correspondentElem = docCard.querySelector('[data-correspondent]');
Expand All @@ -670,7 +696,7 @@ class PlaygroundAnalyzer {
infoContainer.className = 'info-item';

const newValue = document.createElement('span');
newValue.className = 'updated-text truncate';
newValue.className = 'updated-text text-sm truncate';
// Hier müssen wir den neuen Namen aus correspondentNames holen
newValue.textContent = window.correspondentNames?.[analysisResult.correspondent] || analysisResult.correspondent;

Expand All @@ -686,6 +712,33 @@ class PlaygroundAnalyzer {
infoContainer.dataset.correspondent = analysisResult.correspondent;
}
}

// Update document type
if (analysisResult.document_type && analysisResult.document_type !== existing.existingDocumentType) {
const documentTypeElem = docCard.querySelector('[data-document_type]');
if (documentTypeElem) {
// Save name instead of id
const oldDocumentTypeName = documentTypeElem.textContent.trim();
const infoContainer = document.createElement('div');
infoContainer.className = 'info-item';

const newValue = document.createElement('span');
newValue.className = 'updated-text text-sm truncate';
// get new name from document types list
newValue.textContent = window.documentTypes?.[analysisResult.document_type] || analysisResult.document_type;

const oldValue = document.createElement('span');
oldValue.className = 'old-value';
// Use saved name
oldValue.textContent = oldDocumentTypeName;

infoContainer.appendChild(newValue);
infoContainer.appendChild(oldValue);

documentTypeElem.parentNode.replaceChild(infoContainer, documentTypeElem);
infoContainer.dataset.document_type = analysisResult.document_type;
}
}

// Titel aktualisieren
if (analysisResult.title && analysisResult.title !== existing.existingTitle) {
Expand All @@ -708,7 +761,18 @@ class PlaygroundAnalyzer {
titleElem.parentNode.replaceChild(infoContainer, titleElem);
}
}


// Show custom fields
if (analysisResult.custom_fields) {
const infoContainer = docCard.querySelector('.info-item-meta');
for (const key in analysisResult.custom_fields) {
const newField = document.createElement('span');
newField.className = 'updated-text text-sm truncate';
newField.textContent = `${analysisResult.custom_fields[key].field_name}: ${analysisResult.custom_fields[key].value}`;
infoContainer.appendChild(newField);
}
}

// Highlight-Effekt
docCard.classList.add('updated');
setTimeout(() => {
Expand Down
16 changes: 10 additions & 6 deletions routes/setup.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,20 +173,21 @@ router.get('/sampleData/:id', async (req, res) => {
// Documents view route
router.get('/playground', protectApiRoute, async (req, res) => {
try {
//limit documents to 16 items
const numberOfDocuments = 16;
const {
documents,
tagNames,
correspondentNames,
documentTypes,
paperlessUrl
} = await documentsService.getDocumentsWithMetadata();

//limit documents to 16 items
documents.length = 16;
} = await documentsService.getDocumentsWithMetadata(numberOfDocuments);

res.render('playground', {
documents,
tagNames,
correspondentNames,
documentTypes,
paperlessUrl,
version: configFile.PAPERLESS_AI_VERSION || ' '
});
Expand Down Expand Up @@ -1089,7 +1090,10 @@ router.post('/manual/analyze', express.json(), async (req, res) => {

router.post('/manual/playground', express.json(), async (req, res) => {
try {
const { content, existingTags, prompt, documentId } = req.body;
const { content, prompt, documentId } = req.body;
let existingTags = await paperlessService.getTags();
let existingCorrespondentList = await paperlessService.listCorrespondentsNames();
existingCorrespondentList = existingCorrespondentList.map(correspondent => correspondent.name);

if (!content || typeof content !== 'string') {
console.log('Invalid content received:', content);
Expand All @@ -1106,7 +1110,7 @@ router.post('/manual/playground', express.json(), async (req, res) => {
)
return res.json(analyzeDocument);
} else if (process.env.AI_PROVIDER === 'ollama') {
const analyzeDocument = await ollamaService.analyzePlayground(content, prompt);
const analyzeDocument = await ollamaService.analyzePlayground(content, existingTags, existingCorrespondentList, prompt);
return res.json(analyzeDocument);
} else if (process.env.AI_PROVIDER === 'custom') {
const analyzeDocument = await customService.analyzePlayground(content, prompt);
Expand Down
21 changes: 17 additions & 4 deletions services/documentsService.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ class DocumentsService {
constructor() {
this.tagCache = new Map();
this.correspondentCache = new Map();
this.documentTypeCache = new Map();
}

async getTagNames() {
Expand All @@ -27,11 +28,22 @@ class DocumentsService {
return Object.fromEntries(this.correspondentCache);
}

async getDocumentsWithMetadata() {
const [documents, tagNames, correspondentNames] = await Promise.all([
paperlessService.getDocuments(),
async getDocumentTypes() {
if (this.documentTypeCache.size === 0) {
const documentTypes = await paperlessService.listDocumentTypes();
documentTypes.forEach(documentType => {
this.documentTypeCache.set(documentType.id, documentType.name);
})
}
return Object.fromEntries(this.documentTypeCache);
}

async getDocumentsWithMetadata(numberOfDocuments = -1) {
const [documents, tagNames, correspondentNames, documentTypes] = await Promise.all([
paperlessService.getDocuments(numberOfDocuments),
this.getTagNames(),
this.getCorrespondentNames()
this.getCorrespondentNames(),
this.getDocumentTypes()
]);

// Sort documents by created date (newest first)
Expand All @@ -41,6 +53,7 @@ class DocumentsService {
documents,
tagNames,
correspondentNames,
documentTypes,
paperlessUrl: process.env.PAPERLESS_API_URL.replace('/api', '')
};
}
Expand Down
56 changes: 50 additions & 6 deletions services/paperlessService.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -469,6 +469,50 @@ class PaperlessService {
}
}

async listDocumentTypes() {
this.initialize();
let documentTypes = [];
let page = 1;
let hasNextPage = true;

try {
while (hasNextPage) {
const response = await this.client.get('/document_types/', {
params: {
fields: 'id,name',
count: true,
page: page
}
});

const { results, next } = response.data;

documentTypes = documentTypes.concat(
results.map(documentType => ({
name: documentType.name,
id: documentType.id,
document_count: documentType.document_count
}))
);

// Prüfe, ob es eine nächste Seite gibt
hasNextPage = next !== null;
page++;

// Optional: Füge eine kleine Verzögerung hinzu, um die API nicht zu überlasten
if (hasNextPage) {
await new Promise(resolve => setTimeout(resolve, 100));
}
}

return documentTypes;

} catch (error) {
console.error('[ERROR] fetching document types:', error.message);
return [];
}
}

async listTagNames() {
this.initialize();
let allTags = [];
Expand Down Expand Up @@ -506,7 +550,7 @@ class PaperlessService {
}
}

async getAllDocuments() {
async getAllDocuments(numberOfDocuments = -1) {
this.initialize();
if (!this.client) {
console.error('[DEBUG] Client not initialized');
Expand Down Expand Up @@ -549,8 +593,8 @@ class PaperlessService {
try {
const params = {
page,
page_size: 100,
fields: 'id,title,created,created_date,added,tags,correspondent'
page_size: numberOfDocuments > 0 ? numberOfDocuments : 100,
fields: 'id,title,created,created_date,added,tags,correspondent,document_type'
};

// Füge Tag-Filter hinzu, wenn Tags definiert sind
Expand All @@ -570,7 +614,7 @@ class PaperlessService {
}

documents = documents.concat(response.data.results);
hasMore = response.data.next !== null;
hasMore = numberOfDocuments < 0 && response.data.next !== null;
page++;

console.log(
Expand Down Expand Up @@ -763,8 +807,8 @@ class PaperlessService {


// Aktualisierte getDocuments Methode
async getDocuments() {
return this.getAllDocuments();
async getDocuments(numberOfDocuments = -1) {
return this.getAllDocuments(numberOfDocuments);
}

async getDocumentContent(documentId) {
Expand Down
17 changes: 11 additions & 6 deletions views/playground.ejs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,17 +151,22 @@
<h3 class="text-sm font-medium truncate"><%= doc.title %></h3>
</div>

<div class="info-item">
<p class="text-xs text-gray-600 truncate">
<div class="info-item info-item-meta">
<p class="text-xs text-gray-600 truncate"
data-document_date="<%= new Date(doc.created).valueOf() %>">
<%= new Date(doc.created).toLocaleDateString() %>
</p>

<% if (doc.correspondent) { %>
<p class="text-xs text-gray-600 truncate"
<p class="text-xs text-gray-600 truncate"
data-correspondent="<%= doc.correspondent %>">
<%= correspondentNames[doc.correspondent] || 'Unknown' %>
Correspondent: <%= correspondentNames[doc.correspondent] || 'Unknown' %>
</p>

<p class="text-xs text-gray-600 truncate"
data-document_type="<%= doc.document_type %>">
Document type: <%= documentTypes[doc.document_type] || 'Unknown' %>
</p>
<% } %>

</div>
</div>
</div>
Expand Down