admin/apollo-fe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor document ingestion logic and improve UI responsiveness

Browse Source
This commit is contained in:
andrea.terzani
2025-03-22 19:57:37 +01:00
parent 77171af7c2
commit aadf076aca
4 changed files with 443 additions and 276 deletions
Download Patch File Download Diff File Expand all files Collapse all files

181
src/views/pages/ksDocuments/KsDocuments.vue
View File

@@ -1,18 +1,17 @@
<template>
<Toast />
<ConfirmPopup></ConfirmPopup>
<div >
<DataTable v-model:filters="filters" v-model:expandedRows="expandedRows" @rowExpand="onRowExpand"
@rowCollapse="onRowCollapse" :value="ksdocuments"
:loading="loadingStore.loading_exectuion"
:paginator="true" :rows="10"
paginatorTemplate="FirstPageLink PrevPageLink PageLinks NextPageLink LastPageLink CurrentPageReport RowsPerPageDropdown"
currentPageReportTemplate="Showing {first} to {last} of {totalRecords} records"
:rowsPerPageOptions="[10, 15, 20, 50, 100]" dataKey="id" :rowHover="true" rowGroupMode="subheader"
:sortOrder="1" filterDisplay="menu"
:globalFilterFields="['ingestionInfo.metadata.KsApplicationName', 'ingestionInfo.metadata.KsFileSources', 'ingestionInfo.metadata.KsDocSource', 'ingestionStatus', 'ingestionDateFormat']"
tableStyle="min-width: 70rem" removableSort>
<template #header>
<ConfirmPopup></ConfirmPopup>
<div>
<DataTable v-model:filters="filters" v-model:expandedRows="expandedRows" @rowExpand="onRowExpand"
@rowCollapse="onRowCollapse" :value="ksdocuments" :loading="loadingStore.loading_exectuion"
:paginator="true" :rows="10"
paginatorTemplate="FirstPageLink PrevPageLink PageLinks NextPageLink LastPageLink CurrentPageReport RowsPerPageDropdown"
currentPageReportTemplate="Showing {first} to {last} of {totalRecords} records"
:rowsPerPageOptions="[10, 15, 20, 50, 100]" dataKey="id" :rowHover="true" rowGroupMode="subheader"
:sortOrder="1" filterDisplay="menu"
:globalFilterFields="['ingestionInfo.metadata.KsApplicationName', 'ingestionInfo.metadata.KsFileSources', 'ingestionInfo.metadata.KsDocSource', 'ingestionStatus', 'ingestionDateFormat']"
tableStyle="min-width: 70rem" removableSort>
<template #header>
<div class="flex items-center justify-between gap-4 p-4 ">
<span class="text-xl font-bold">Knowledge Source Documents</span>
<div class="flex items-center gap-2 flex-grow">
@@ -23,19 +22,18 @@
<InputText v-model="filters['global'].value" placeholder="Search..." />
</IconField>
</div>
<Button icon="pi pi-plus" rounded raised @click="newKsDocument()"
v-tooltip.left="'Add New Document'"
class="mr-2 p-button-sm" />
<Button icon="pi pi-check-circle" rounded raised @click="startlngestion()"
v-tooltip.left="'Start All documents Ingestion'"
class="mr-8 p-button-sm"
:disabled="allDocumentsIngested" :class="{ 'p-button': allDocumentsIngested }" />
<Button icon="pi pi-plus" rounded raised @click="newKsDocument()"
v-tooltip.left="'Add New Document'" class="mr-2 p-button-sm" />
<!--<Button icon="pi pi-check-circle" rounded raised @click="startlngestion()"
v-tooltip.left="'Start All documents Ingestion'" class="mr-8 p-button-sm"
:disabled="allDocumentsIngested" :class="{ 'p-button': allDocumentsIngested }" />-->
</div>
</template>
<template #empty>No Records found</template>
<Column field="ingestionStatus" header="Status" sortable>
<template #body="slotProps">
<Tag :value="slotProps.data.ingestionStatus" :severity="getStatus(slotProps.data)" />
<Tag v-tooltip="slotProps.data.ingestionMessage" :value="slotProps.data.ingestionStatus"
:severity="getStatus(slotProps.data)" />
</template>
<template #filter="{ filterModel, filterCallback }">
<Select v-model="filterModel.value" @change="filterCallback()" :options="statuses"
@@ -48,7 +46,10 @@
</Column>
<Column field="ingestionInfo.metadata.KsFileSource" header="FileName" sortable>
<template #body="{ data }">
{{ data.fileName }}
<span>
<i :class="getFileIcon(data.fileName)" class="mr-2"></i>
{{ data.fileName }}
</span>
</template>
<template #filter="{ filterModel, filterCallback }">
<InputText v-model="filterModel.value" type="text" @input="filterCallback()"
@@ -83,30 +84,29 @@
@change="updateFilterModel" />
</template>
</Column>
<Column field="id" :style="{ position: 'sticky', right: '0', zIndex: '1', background: '#f3f3f3' }">
<!-- <Column field="id" :style="{ position: 'sticky', right: '0', zIndex: '1'}"> -->
<template #body="slotProps">
<div class="flex justify-center items-center space-x-3">
<!-- PARTE EDIT DA CREARE -->
<!-- <Button type="button" icon="pi pi-pencil" rounded @click="editKsDocument(slotProps.data)"
v-tooltip="'Edit the information of document'" /> -->
<Button type="button" icon="pi pi-play" rounded @click="startIndividualngestion(slotProps.data.id)"
<Column field="id" header="Actions" style="width: 10rem" class="text-center">
<template #body="slotProps">
<div class="flex justify-center items-center space-x-3">
<Button type="button" icon="pi pi-play" rounded
@click="startIndividualngestion(slotProps.data.id)"
v-tooltip.left="'Start Ingestion of document'"
:disabled="slotProps.data.ingestionStatus === 'INGESTED'"
:class="{ 'p-button': slotProps.data.ingestionStatus === 'INGESTED' }" />
<Button type="button" icon="pi pi-trash" rounded @click="confirmDeleteFromVectorStore(slotProps.data.id)"
:class="{ 'p-button': slotProps.data.ingestionStatus === 'INGESTED' }"></Button>
<Button type="button" icon="pi pi-trash" rounded
@click="confirmDeleteFromVectorStore(slotProps.data.id)"
v-tooltip.left="'Delete the ingested Record'"
:disabled="slotProps.data.ingestionStatus !== 'INGESTED'"
:class="{ 'mr-2 p-button': slotProps.data.ingestionStatus !== 'INGESTED' }" />
:disabled="slotProps.data.ingestionStatus !== 'INGESTED' || slotProps.data.ingestionStatus == 'IN PROGRESS'"
:class="{ 'mr-2 p-button': slotProps.data.ingestionStatus !== 'INGESTED' }"></Button>
<Button type="button" icon="pi pi-search" rounded @click="openSimilaritySearch(slotProps.data)"
v-tooltip.left="'Similarity Search'"
:disabled="slotProps.data.ingestionStatus !== 'INGESTED'"
:class="{ 'mr-2 p-button': slotProps.data.ingestionStatus !== 'INGESTED' }" />
v-tooltip.left="'Similarity Search'"
:disabled="slotProps.data.ingestionStatus !== 'INGESTED' || slotProps.data.ingestionStatus == 'IN PROGRESS'"
:class="{ 'mr-2 p-button': slotProps.data.ingestionStatus !== 'INGESTED' }"></Button>
<Button type="button" icon="pi pi-download" rounded @click="downloadFile(slotProps.data)"
v-tooltip.left="'Download file'"
class="mr-2 p-button" />
</div>
</template>
v-tooltip.left="'Download file'" class="mr-2 p-button"></Button>
</div>
</template>
</Column>
</DataTable>
@@ -126,9 +126,7 @@ import Button from 'primevue/button';
import Column from 'primevue/column';
import DataTable from 'primevue/datatable';
import DatePicker from 'primevue/datepicker';
import Dialog from 'primevue/dialog';
import InputText from 'primevue/inputtext';
import ProgressSpinner from 'primevue/progressspinner';
import Select from 'primevue/select';
import Tag from 'primevue/tag';
import { KsDocumentService } from '../../../service/KsDocumentService';
@@ -164,51 +162,50 @@ const initFilters = () => {
initFilters();
const statuses = ref(['NEW', 'LOADED','INGESTED', 'FAILED']); // Add your statuses here
const statuses = ref(['NEW', 'LOADED', 'INGESTED', 'FAILED', 'ERROR']); // Add your statuses here
onMounted(() => {
userPrefStore.fetchUserData().then(() => {
updateDocuments();
setInterval(() => {
updateDocuments();
}, 10000);
});
});
const onRowExpand = (event) => {
console.log("Row expanded:", event.data);
console.log("Row expanded:", event.data);
};
const onRowCollapse = (event) => {
console.log("Row collapsed:", event.data);
console.log("Row collapsed:", event.data);
};
watch(() => userPrefStore.getSelApp, updateDocuments, { immediate: true });
function updateDocuments() {
//loading.value = true;
ksDocumentStore.fetchKsDocument().then(() => {
ksdocuments.value = getCustomDatewithAllResponse();
//ingestionDialogVisible.value = false;
//loading.value = false;
});
};
// Computed property to check if all documents are ingested
const allDocumentsIngested = computed(() => {
return ksdocuments.value && ksdocuments.value.every(doc => doc.ingestionStatus === 'INGESTED');
});
const getStatus = (data) => {
if (data.ingestionStatus === 'INGESTED') {
return 'success';
} else if (data.ingestionStatus === 'LOADED') {
} else if (data.ingestionStatus === 'LOADED' || data.ingestionStatus === 'ERROR') {
return 'danger';
} else if (data.ingestionStatus === 'IN PROGRESS') {
return 'info';
} else {
return 'warn';
return 'warning';
}
}
@@ -228,18 +225,11 @@ const filename = ref("");
// Funzione per scaricare il file
const downloadFile = async (doc) => {
/*if (!filename.value) {
alert("Inserisci il nome del file.");
return;
}*/
console.log("doc", doc)
try {
//const response = await axios.post('/fe-api/ksdocuments/downloadKSDocument', doc, { responseType: "blob", });
const response = await KsDocumentService.downloadKsDocument(doc);
console.log("response download", response);
const contentType = response.headers['content-type']; // Tipo MIME dinamico
@@ -253,21 +243,11 @@ const downloadFile = async (doc) => {
const fileName = doc.fileName;
console.log("fileName", fileName)
// Imposta il nome e avvia il download
link.setAttribute('download', fileName);
document.body.appendChild(link);
link.click();
link.remove();
// Crea un URL temporaneo per il download del file
/*console.log("response download", response);
const url = window.URL.createObjectURL(new Blob([response.data]));
const link = document.createElement("a");
link.href = url;
link.setAttribute("download", filename.value); // Imposta il nome del file
document.body.appendChild(link);
link.click();
link.remove();*/
} catch (error) {
console.error("Error while downloading the file:", error);
toast.add({ severity: 'error', summary: 'Error', detail: 'Error while downloading the file. Check the file name.', life: 3000 });
@@ -293,7 +273,7 @@ const confirmDeleteFromVectorStore = (id) => {
icon: 'pi pi-exclamation-triangle',
rejectProps: {
label: 'Cancel',
class:"p-button",
class: "p-button",
outlined: true,
},
acceptProps: {
@@ -319,7 +299,12 @@ const confirmDeleteFromVectorStore = (id) => {
axios.post('/fe-api/vector-store/deleteRecordsFromVectorStore', requestPayload)
.then(response => {
console.log('Delete resource:', response.data)
updateDocuments(); // Call updateDocuments after successful deletion
ksdocuments.value.forEach(element => {
if (element.id == id) {
element.ingestionStatus = "LOADED"
console.log("Updated element", element)
}
});
toast.add({ severity: 'success', summary: 'Success', detail: 'Deletion Completed', life: 3000 });
})
.catch(error => {
@@ -335,26 +320,24 @@ const startIndividualngestion = (id) => {
toast.add({ severity: 'info', summary: 'Info', detail: 'Starting Ingestion', life: 3000 });
axios.get(`/test/ingest_document/${id}`)
.then(response => {
ingestionResult.value = response.data;
if (response.data.status == "OK") {
ksdocuments.value.forEach(element => {
updateDocuments();
if (response.data.ingestedDocumentId.includes(element.id)) {
element.status = "INGESTED"
updateDocuments();
toast.add({severity: 'success', summary: 'Success', detail: 'File ingested successfully!', life: 3000});
}
});
} else {
ingestionResult.value = `Error: ${response.data.message}`;
}
toast.add({ severity: 'success', summary: 'Success', detail: 'Document ingestion started...', life: 3000 });
//ingestionDialogVisible.value = true;
}
if (response.data.status == "ERROR") {
toast.add({ severity: 'error', summary: 'Success', detail: 'Error ingesting document:' + response.data.message, life: 3000 });
}
ksdocuments.value.forEach(element => {
if (element.id == id) {
element.ingestionStatus = response.data.status
console.log("Updated element", element)
}
});
})
.catch(error => {
//ingestionDialogVisible.value = true;
console.error('Error ingesting record: ', error)
toast.add({ severity: 'error', summary: 'Error', detail: 'Error while ingesting', life: 3000 });
toast.add({ severity: 'error', summary: 'Success', detail: 'Error ingesting document:' + error, life: 3000 });
});
};
@@ -384,12 +367,27 @@ const startlngestion = () => {
//new record creation
const newKsDocument = () => {
router.push({ name: 'ks-document-new' });
};
const getFileIcon = (filename) => {
const ext = filename.split('.').pop();
if (ext === 'pdf') {
return 'pi pi-file-pdf';
} else if (ext === 'doc' || ext === 'docx') {
return 'pi pi-file-word';
} else if (ext === 'xls' || ext === 'xlsx') {
return 'pi pi-file-excel';
} else if (ext === 'ppt' || ext === 'pptx') {
return 'pi pi-file-powerpoint';
} else {
return 'pi pi-file';
}
}
// Function to format date string
function formatDate(dateString) {
// Parse the date string using moment
return moment(dateString).format('MM/DD/YYYY');
return moment(dateString).format('DD/MM/YYYY');
}
@@ -463,5 +461,4 @@ function formatDate(dateString) {
.card {
margin: 2rem;
}
</style>