admin/apollo-fe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor document ingestion logic and improve UI responsiveness

Browse Source
This commit is contained in:
andrea.terzani
2025-03-22 19:57:37 +01:00
parent 77171af7c2
commit aadf076aca
4 changed files with 443 additions and 276 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/main.js
View File

@@ -14,7 +14,7 @@ import '@/assets/tailwind.css';
import Nora from '@primevue/themes/nora';
import axios from 'axios';
import { LoadingStore } from './stores/LoadingStore.js';
axios.defaults.baseURL = import.meta.env.VITE_BACKEND_URL; //'http://localhost:8082'
axios.defaults.baseURL = import.meta.env.VITE_BACKEND_URL;//'http://localhost:8082' //'
console.log(import.meta.env.VITE_BACKEND_URL);

181
src/views/pages/ksDocuments/KsDocuments.vue
View File

@@ -1,18 +1,17 @@
<template>
<Toast />
<ConfirmPopup></ConfirmPopup>
<div >
<DataTable v-model:filters="filters" v-model:expandedRows="expandedRows" @rowExpand="onRowExpand"
@rowCollapse="onRowCollapse" :value="ksdocuments"
:loading="loadingStore.loading_exectuion"
:paginator="true" :rows="10"
paginatorTemplate="FirstPageLink PrevPageLink PageLinks NextPageLink LastPageLink CurrentPageReport RowsPerPageDropdown"
currentPageReportTemplate="Showing {first} to {last} of {totalRecords} records"
:rowsPerPageOptions="[10, 15, 20, 50, 100]" dataKey="id" :rowHover="true" rowGroupMode="subheader"
:sortOrder="1" filterDisplay="menu"
:globalFilterFields="['ingestionInfo.metadata.KsApplicationName', 'ingestionInfo.metadata.KsFileSources', 'ingestionInfo.metadata.KsDocSource', 'ingestionStatus', 'ingestionDateFormat']"
tableStyle="min-width: 70rem" removableSort>
<template #header>
<ConfirmPopup></ConfirmPopup>
<div>
<DataTable v-model:filters="filters" v-model:expandedRows="expandedRows" @rowExpand="onRowExpand"
@rowCollapse="onRowCollapse" :value="ksdocuments" :loading="loadingStore.loading_exectuion"
:paginator="true" :rows="10"
paginatorTemplate="FirstPageLink PrevPageLink PageLinks NextPageLink LastPageLink CurrentPageReport RowsPerPageDropdown"
currentPageReportTemplate="Showing {first} to {last} of {totalRecords} records"
:rowsPerPageOptions="[10, 15, 20, 50, 100]" dataKey="id" :rowHover="true" rowGroupMode="subheader"
:sortOrder="1" filterDisplay="menu"
:globalFilterFields="['ingestionInfo.metadata.KsApplicationName', 'ingestionInfo.metadata.KsFileSources', 'ingestionInfo.metadata.KsDocSource', 'ingestionStatus', 'ingestionDateFormat']"
tableStyle="min-width: 70rem" removableSort>
<template #header>
<div class="flex items-center justify-between gap-4 p-4 ">
<span class="text-xl font-bold">Knowledge Source Documents</span>
<div class="flex items-center gap-2 flex-grow">
@@ -23,19 +22,18 @@
<InputText v-model="filters['global'].value" placeholder="Search..." />
</IconField>
</div>
<Button icon="pi pi-plus" rounded raised @click="newKsDocument()"
v-tooltip.left="'Add New Document'"
class="mr-2 p-button-sm" />
<Button icon="pi pi-check-circle" rounded raised @click="startlngestion()"
v-tooltip.left="'Start All documents Ingestion'"
class="mr-8 p-button-sm"
:disabled="allDocumentsIngested" :class="{ 'p-button': allDocumentsIngested }" />
<Button icon="pi pi-plus" rounded raised @click="newKsDocument()"
v-tooltip.left="'Add New Document'" class="mr-2 p-button-sm" />
<!--<Button icon="pi pi-check-circle" rounded raised @click="startlngestion()"
v-tooltip.left="'Start All documents Ingestion'" class="mr-8 p-button-sm"
:disabled="allDocumentsIngested" :class="{ 'p-button': allDocumentsIngested }" />-->
</div>
</template>
<template #empty>No Records found</template>
<Column field="ingestionStatus" header="Status" sortable>
<template #body="slotProps">
<Tag :value="slotProps.data.ingestionStatus" :severity="getStatus(slotProps.data)" />
<Tag v-tooltip="slotProps.data.ingestionMessage" :value="slotProps.data.ingestionStatus"
:severity="getStatus(slotProps.data)" />
</template>
<template #filter="{ filterModel, filterCallback }">
<Select v-model="filterModel.value" @change="filterCallback()" :options="statuses"
@@ -48,7 +46,10 @@
</Column>
<Column field="ingestionInfo.metadata.KsFileSource" header="FileName" sortable>
<template #body="{ data }">
{{ data.fileName }}
<span>
<i :class="getFileIcon(data.fileName)" class="mr-2"></i>
{{ data.fileName }}
</span>
</template>
<template #filter="{ filterModel, filterCallback }">
<InputText v-model="filterModel.value" type="text" @input="filterCallback()"
@@ -83,30 +84,29 @@
@change="updateFilterModel" />
</template>
</Column>
<Column field="id" :style="{ position: 'sticky', right: '0', zIndex: '1', background: '#f3f3f3' }">
<!-- <Column field="id" :style="{ position: 'sticky', right: '0', zIndex: '1'}"> -->
<template #body="slotProps">
<div class="flex justify-center items-center space-x-3">
<!-- PARTE EDIT DA CREARE -->
<!-- <Button type="button" icon="pi pi-pencil" rounded @click="editKsDocument(slotProps.data)"
v-tooltip="'Edit the information of document'" /> -->
<Button type="button" icon="pi pi-play" rounded @click="startIndividualngestion(slotProps.data.id)"
<Column field="id" header="Actions" style="width: 10rem" class="text-center">
<template #body="slotProps">
<div class="flex justify-center items-center space-x-3">
<Button type="button" icon="pi pi-play" rounded
@click="startIndividualngestion(slotProps.data.id)"
v-tooltip.left="'Start Ingestion of document'"
:disabled="slotProps.data.ingestionStatus === 'INGESTED'"
:class="{ 'p-button': slotProps.data.ingestionStatus === 'INGESTED' }" />
<Button type="button" icon="pi pi-trash" rounded @click="confirmDeleteFromVectorStore(slotProps.data.id)"
:class="{ 'p-button': slotProps.data.ingestionStatus === 'INGESTED' }"></Button>
<Button type="button" icon="pi pi-trash" rounded
@click="confirmDeleteFromVectorStore(slotProps.data.id)"
v-tooltip.left="'Delete the ingested Record'"
:disabled="slotProps.data.ingestionStatus !== 'INGESTED'"
:class="{ 'mr-2 p-button': slotProps.data.ingestionStatus !== 'INGESTED' }" />
:disabled="slotProps.data.ingestionStatus !== 'INGESTED' || slotProps.data.ingestionStatus == 'IN PROGRESS'"
:class="{ 'mr-2 p-button': slotProps.data.ingestionStatus !== 'INGESTED' }"></Button>
<Button type="button" icon="pi pi-search" rounded @click="openSimilaritySearch(slotProps.data)"
v-tooltip.left="'Similarity Search'"
:disabled="slotProps.data.ingestionStatus !== 'INGESTED'"
:class="{ 'mr-2 p-button': slotProps.data.ingestionStatus !== 'INGESTED' }" />
v-tooltip.left="'Similarity Search'"
:disabled="slotProps.data.ingestionStatus !== 'INGESTED' || slotProps.data.ingestionStatus == 'IN PROGRESS'"
:class="{ 'mr-2 p-button': slotProps.data.ingestionStatus !== 'INGESTED' }"></Button>
<Button type="button" icon="pi pi-download" rounded @click="downloadFile(slotProps.data)"
v-tooltip.left="'Download file'"
class="mr-2 p-button" />
</div>
</template>
v-tooltip.left="'Download file'" class="mr-2 p-button"></Button>
</div>
</template>
</Column>
</DataTable>
@@ -126,9 +126,7 @@ import Button from 'primevue/button';
import Column from 'primevue/column';
import DataTable from 'primevue/datatable';
import DatePicker from 'primevue/datepicker';
import Dialog from 'primevue/dialog';
import InputText from 'primevue/inputtext';
import ProgressSpinner from 'primevue/progressspinner';
import Select from 'primevue/select';
import Tag from 'primevue/tag';
import { KsDocumentService } from '../../../service/KsDocumentService';
@@ -164,51 +162,50 @@ const initFilters = () => {
initFilters();
const statuses = ref(['NEW', 'LOADED','INGESTED', 'FAILED']); // Add your statuses here
const statuses = ref(['NEW', 'LOADED', 'INGESTED', 'FAILED', 'ERROR']); // Add your statuses here
onMounted(() => {
userPrefStore.fetchUserData().then(() => {
updateDocuments();
setInterval(() => {
updateDocuments();
}, 10000);
});
});
const onRowExpand = (event) => {
console.log("Row expanded:", event.data);
console.log("Row expanded:", event.data);
};
const onRowCollapse = (event) => {
console.log("Row collapsed:", event.data);
console.log("Row collapsed:", event.data);
};
watch(() => userPrefStore.getSelApp, updateDocuments, { immediate: true });
function updateDocuments() {
//loading.value = true;
ksDocumentStore.fetchKsDocument().then(() => {
ksdocuments.value = getCustomDatewithAllResponse();
//ingestionDialogVisible.value = false;
//loading.value = false;
});
};
// Computed property to check if all documents are ingested
const allDocumentsIngested = computed(() => {
return ksdocuments.value && ksdocuments.value.every(doc => doc.ingestionStatus === 'INGESTED');
});
const getStatus = (data) => {
if (data.ingestionStatus === 'INGESTED') {
return 'success';
} else if (data.ingestionStatus === 'LOADED') {
} else if (data.ingestionStatus === 'LOADED' || data.ingestionStatus === 'ERROR') {
return 'danger';
} else if (data.ingestionStatus === 'IN PROGRESS') {
return 'info';
} else {
return 'warn';
return 'warning';
}
}
@@ -228,18 +225,11 @@ const filename = ref("");
// Funzione per scaricare il file
const downloadFile = async (doc) => {
/*if (!filename.value) {
alert("Inserisci il nome del file.");
return;
}*/
console.log("doc", doc)
try {
//const response = await axios.post('/fe-api/ksdocuments/downloadKSDocument', doc, { responseType: "blob", });
const response = await KsDocumentService.downloadKsDocument(doc);
console.log("response download", response);
const contentType = response.headers['content-type']; // Tipo MIME dinamico
@@ -253,21 +243,11 @@ const downloadFile = async (doc) => {
const fileName = doc.fileName;
console.log("fileName", fileName)
// Imposta il nome e avvia il download
link.setAttribute('download', fileName);
document.body.appendChild(link);
link.click();
link.remove();
// Crea un URL temporaneo per il download del file
/*console.log("response download", response);
const url = window.URL.createObjectURL(new Blob([response.data]));
const link = document.createElement("a");
link.href = url;
link.setAttribute("download", filename.value); // Imposta il nome del file
document.body.appendChild(link);
link.click();
link.remove();*/
} catch (error) {
console.error("Error while downloading the file:", error);
toast.add({ severity: 'error', summary: 'Error', detail: 'Error while downloading the file. Check the file name.', life: 3000 });
@@ -293,7 +273,7 @@ const confirmDeleteFromVectorStore = (id) => {
icon: 'pi pi-exclamation-triangle',
rejectProps: {
label: 'Cancel',
class:"p-button",
class: "p-button",
outlined: true,
},
acceptProps: {
@@ -319,7 +299,12 @@ const confirmDeleteFromVectorStore = (id) => {
axios.post('/fe-api/vector-store/deleteRecordsFromVectorStore', requestPayload)
.then(response => {
console.log('Delete resource:', response.data)
updateDocuments(); // Call updateDocuments after successful deletion
ksdocuments.value.forEach(element => {
if (element.id == id) {
element.ingestionStatus = "LOADED"
console.log("Updated element", element)
}
});
toast.add({ severity: 'success', summary: 'Success', detail: 'Deletion Completed', life: 3000 });
})
.catch(error => {
@@ -335,26 +320,24 @@ const startIndividualngestion = (id) => {
toast.add({ severity: 'info', summary: 'Info', detail: 'Starting Ingestion', life: 3000 });
axios.get(`/test/ingest_document/${id}`)
.then(response => {
ingestionResult.value = response.data;
if (response.data.status == "OK") {
ksdocuments.value.forEach(element => {
updateDocuments();
if (response.data.ingestedDocumentId.includes(element.id)) {
element.status = "INGESTED"
updateDocuments();
toast.add({severity: 'success', summary: 'Success', detail: 'File ingested successfully!', life: 3000});
}
});
} else {
ingestionResult.value = `Error: ${response.data.message}`;
}
toast.add({ severity: 'success', summary: 'Success', detail: 'Document ingestion started...', life: 3000 });
//ingestionDialogVisible.value = true;
}
if (response.data.status == "ERROR") {
toast.add({ severity: 'error', summary: 'Success', detail: 'Error ingesting document:' + response.data.message, life: 3000 });
}
ksdocuments.value.forEach(element => {
if (element.id == id) {
element.ingestionStatus = response.data.status
console.log("Updated element", element)
}
});
})
.catch(error => {
//ingestionDialogVisible.value = true;
console.error('Error ingesting record: ', error)
toast.add({ severity: 'error', summary: 'Error', detail: 'Error while ingesting', life: 3000 });
toast.add({ severity: 'error', summary: 'Success', detail: 'Error ingesting document:' + error, life: 3000 });
});
};
@@ -384,12 +367,27 @@ const startlngestion = () => {
//new record creation
const newKsDocument = () => {
router.push({ name: 'ks-document-new' });
};
const getFileIcon = (filename) => {
const ext = filename.split('.').pop();
if (ext === 'pdf') {
return 'pi pi-file-pdf';
} else if (ext === 'doc' || ext === 'docx') {
return 'pi pi-file-word';
} else if (ext === 'xls' || ext === 'xlsx') {
return 'pi pi-file-excel';
} else if (ext === 'ppt' || ext === 'pptx') {
return 'pi pi-file-powerpoint';
} else {
return 'pi pi-file';
}
}
// Function to format date string
function formatDate(dateString) {
// Parse the date string using moment
return moment(dateString).format('MM/DD/YYYY');
return moment(dateString).format('DD/MM/YYYY');
}
@@ -463,5 +461,4 @@ function formatDate(dateString) {
.card {
margin: 2rem;
}
</style>

76
src/views/pages/ksDocuments/KsNewDocumentForm.vue
View File

@@ -32,15 +32,14 @@
</FileUpload>
</div>
<div class="flex mt-16 pt-6 justify-end">
<Button label="Next" icon="pi pi-arrow-right" @click="activateCallback('2')"
:disabled="!isFileSelected"
<Button label="Next" icon="pi pi-arrow-right" @click="activateCallback('2')" :disabled="!isFileSelected"
style="width: 150px;" class="p-button" v-tooltip.top="!isFileSelected ? 'Please select a file' : ''" />
</div>
</StepPanel>
<form @submit.prevent="submitForm" class="p-fluid">
<StepPanel value="2" v-slot="{ activateCallback }">
<div class="flex flex-col h-full">
<Tabs class="mt-6">
<Tabs class="mt-6" value="0">
<TabList>
<Tab value="0">Base</Tab>
<Tab value="1">Advanced</Tab>
@@ -75,7 +74,7 @@
<span class="p-float-label">
<label for="ksApplicationName">KS Application Name</label>
<Dropdown id="ksApplicationName" v-model="formData.ksApplicationName"
:options="availableAppOptions" optionLabel="name" required class="w-full"></Dropdown>
:options="availableAppOptions" optionLabel="name" required class="w-full"></Dropdown>
</span>
</div>
<div class="col-12 md:col-6 mb-4">
@@ -125,7 +124,7 @@
class="w-full" />
</span>
<small v-if="errors.minChunkSizeToEmbed" class="text-red-500">{{ errors.minChunkSizeToEmbed
}}</small>
}}</small>
</div>
<div class="col-12 md:col-6 mb-4">
<span class="p-float-label">
@@ -136,7 +135,7 @@
class="w-full" />
</span>
<small v-if="errors.maxNumberOfChunks" class="text-red-500">{{ errors.maxNumberOfChunks
}}</small>
}}</small>
</div>
</TabPanel>
</TabPanels>
@@ -156,7 +155,7 @@
<p v-if="selectedFile" class="text-center">You can execute the ingestion for the following file:</p>
<p v-if="selectedFile" class="text-center">{{ selectedFile.name }}</p>
<div class="flex justify-center mt-4">
<Button :disabled="loadingStore.isLoading" type="submit" label="Ingest" :fluid="false"></Button>
<Button type="submit" label="Ingest" :fluid="false"></Button>
</div>
</div>
<div class="flex justify-between pt-6">
@@ -171,27 +170,28 @@
</div>
</Fluid>
<Dialog v-model:visible="showDialog" header="Confirm Overwrite" :modal="true">
<p>The document "{{ selectedFile?.name }}" already exists with status "{{ existingDocument?.ingestionStatus }}". Do you want to overwrite it?</p>
<p>The document "{{ selectedFile?.name }}" already exists with status "{{ existingDocument?.ingestionStatus }}". Do
you want to overwrite it?</p>
<template #footer>
<Button label="Cancel" icon="pi pi-times" @click="cancelOverwrite" class="p-button" />
<Button label="Overwrite" icon="pi pi-check" @click="confirmOverwrite" class="p-button-danger"/>
<Button label="Overwrite" icon="pi pi-check" @click="confirmOverwrite" class="p-button-danger" />
</template>
</Dialog>
</template>
<script setup>
import axios from 'axios';
import { useToast } from 'primevue/usetoast';
import { ref, watch, computed, onMounted, nextTick } from 'vue';
import { useRouter } from 'vue-router';
import { UserPrefStore } from '../../../stores/UserPrefStore';
import Stepper from 'primevue/stepper';
import StepList from 'primevue/steplist';
import StepPanels from 'primevue/steppanels';
import Step from 'primevue/step';
import StepList from 'primevue/steplist';
import StepPanel from 'primevue/steppanel';
import { LoadingStore } from '../../../stores/LoadingStore';
import StepPanels from 'primevue/steppanels';
import Stepper from 'primevue/stepper';
import { useToast } from 'primevue/usetoast';
import { computed, ref, watch } from 'vue';
import { useRouter } from 'vue-router';
import { KsDocumentStore } from '../../../stores/KsDocumentStore';
import { LoadingStore } from '../../../stores/LoadingStore';
import { UserPrefStore } from '../../../stores/UserPrefStore';
const toast = useToast();
const router = useRouter();
@@ -285,6 +285,7 @@ const onFileSelect = (event) => {
if (event.files && event.files.length > 0) {
selectedFile.value = event.files[0];
formData.value.type = selectedFile.value.name.split('.').pop().toLowerCase();
formData.value.description = selectedFile.value.name;
fileUploaded.value = true;
} else {
selectedFile.value = null;
@@ -319,22 +320,29 @@ const onFileRemove = () => {
};
const startIndividualngestion = async (id) => {
try {
const response = await axios.get(`/test/ingest_document/${id}`);
ingestionResult.value = response.data;
toast.add({ severity: 'info', summary: 'Info', detail: 'Starting Ingestion', life: 3000 });
axios.get(`/test/ingest_document/${id}`)
.then(response => {
if (response.data.status == "OK") {
toast.add({ severity: 'success', summary: 'Success', detail: 'Document ingestion started...', life: 3000 });
}
if (response.data.status == "ERROR") {
toast.add({ severity: 'error', summary: 'Success', detail: 'Error ingesting document:' + response.data.message, life: 3000 });
}
})
.catch(error => {
//ingestionDialogVisible.value = true;
console.error('Error ingesting record: ', error)
toast.add({ severity: 'error', summary: 'Success', detail: 'Error ingesting document:' + error, life: 3000 });
});
if (response.status !== 200) {
ingestionResult.value = `Error: ${response.data.message}`;
}
ingestionDialogVisible.value = true;
} catch (error) {
console.error('Error during ingestion:', error);
ingestionDialogVisible.value = true;
throw error;
}
};
const deleteRecord = (name) => {
const deleteRecord = (name) => {
const documentToDelete = ksdocuments.value.find(doc => doc.name === name);
if (!documentToDelete) {
console.error('Document not found');
@@ -409,12 +417,14 @@ const uploadFile = async (formDataToSend) => {
'Content-Type': 'multipart/form-data'
}
});
await startIndividualngestion(response.data.id);
toast.add({ severity: 'success', summary: 'Success', detail: 'File ingested successfully', life: 3000 });
ksdocuments.value.push(response.data);
startIndividualngestion(response.data.id);
router.push({ name: 'ks-document' });
} catch (error) {
console.error('Ingestion failed:', error);
toast.add({ severity: 'error', summary: 'Error', detail: 'File ingestion failed', life: 3000 });
}
};