admin/apollo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Deletion of data from collections implemented

Browse Source
This commit is contained in:
sumedh
2024-08-08 10:36:38 +05:30
parent bee1dad131
commit 0ce79eca38
9 changed files with 109 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/main/java/com/olympus/apollo/controllers/FeApi/VectorStoreController.java
View File

@@ -1,13 +1,14 @@
package com.olympus.apollo.controllers.FeApi;
import com.olympus.apollo.dto.DeletionRequest;
import com.olympus.apollo.dto.VectorStoreDetails;
import com.olympus.apollo.models.VectorStore;
import com.olympus.apollo.repository.VectorStoreRepository;
import com.olympus.apollo.services.DeletionService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import java.util.List;
import java.util.stream.Collectors;
@RestController
@RequestMapping("/fe-api/vector-store")
@@ -16,6 +17,9 @@ public class VectorStoreController {
@Autowired
private VectorStoreRepository vectorStoreRepository;
@Autowired
private DeletionService deletionService;
@GetMapping("")
public List<VectorStore> getAllVectorStore(){
List<VectorStore> result = (List<VectorStore>) vectorStoreRepository.findAll();
@@ -39,4 +43,10 @@ public class VectorStoreController {
return vectorStoreRepository.findAllDetails();
}
@PostMapping("/deleteRecords")
public String deleteRecords(@RequestBody DeletionRequest deletionRequest){
deletionService.deleteRecords(deletionRequest);
return "Records Deleted Successfully";
}
}

2
src/main/java/com/olympus/apollo/controllers/KSFileController.java
View File

@@ -53,7 +53,7 @@ public class KSFileController {
metadata.put("KsApplicationName", fileUploadDTO.getKsApplicationName());
metadata.put("KsDoctype", fileUploadDTO.getKsDocType());
metadata.put("KsDocSource", fileUploadDTO.getKsDocSource());
metadata.put("Source", file.getOriginalFilename());
metadata.put("KsFileSource", file.getOriginalFilename());
ksIngestionInfo.setMetadata(metadata);
ksIngestionInfo.setDefaultChunkSize(fileUploadDTO.getDefaultChunkSize());

4
src/main/java/com/olympus/apollo/controllers/TestController.java
View File

@@ -34,8 +34,8 @@ public class TestController {
}
@GetMapping("test/query_vector")
public List<String> testSimilaritySearch(@RequestParam String query, @RequestParam String type) {
return ksIngestor.testSimilaritySearch(query,type);
public List<String> testSimilaritySearch(@RequestParam String query, @RequestParam String filterQuery) {
return ksIngestor.testSimilaritySearch(query,filterQuery);
}
@GetMapping("test/delete")

15
src/main/java/com/olympus/apollo/dto/DeletionRequest.java Normal file
View File

@@ -0,0 +1,15 @@
package com.olympus.apollo.dto;
import lombok.Getter;
import lombok.Setter;
@Getter @Setter
public class DeletionRequest {
private String ksDocumentId;
private String ksIngestionInfoId;
private String ksDoctype;
private String ksDocSource;
private String ksFileSource;
private String ksApplicationName;
}

5
src/main/java/com/olympus/apollo/dto/VectorStoreMetadataDetails.java
View File

@@ -9,7 +9,6 @@ public class VectorStoreMetadataDetails {
private String KsApplicationName;
private String KsDoctype;
private String KsDocSource;
private String source;
private String Source;
private String KsFileSource;
}

5
src/main/java/com/olympus/apollo/repository/VectorStoreRepository.java
View File

@@ -1,10 +1,10 @@
package com.olympus.apollo.repository;
import com.olympus.apollo.dto.VectorStoreDetails;
import com.olympus.apollo.dto.VectorStoreMetadataDetails;
import com.olympus.apollo.models.VectorStore;
import org.springframework.data.mongodb.repository.Aggregation;
import org.springframework.data.mongodb.repository.MongoRepository;
import org.springframework.data.mongodb.repository.Query;
import org.springframework.stereotype.Repository;
import java.util.List;
@@ -21,4 +21,7 @@ public interface VectorStoreRepository extends MongoRepository<VectorStore, Stri
"{ $project: { _id: 1, metadata: 1 } }"
})
List<VectorStoreDetails> findAllDetails();
@Query("{'metadata.KsDoctype': ?0, 'metadata.KsDocSource': ?1, 'metadata.KsFileSource': ?2, 'metadata.KsApplicationName': ?3}")
List<VectorStore> findByMetadata(String ksDoctype, String ksDocSource, String ksFileSource, String ksApplicationName);
}

70
src/main/java/com/olympus/apollo/services/DeletionService.java Normal file
View File

@@ -0,0 +1,70 @@
package com.olympus.apollo.services;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.olympus.apollo.dto.DeletionRequest;
import com.olympus.apollo.dto.VectorStoreMetadataDetails;
import com.olympus.apollo.models.VectorStore;
import com.olympus.apollo.repository.KSDocumentRepository;
import com.olympus.apollo.repository.KSIngestionInfoRepository;
import com.olympus.apollo.repository.VectorStoreRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.List;
@Service
public class DeletionService {
private static final Logger logger = LoggerFactory.getLogger(DeletionService.class);
@Autowired
private KSDocumentRepository ksDocumentRepository;
@Autowired
private KSIngestionInfoRepository ksIngestionInfoRepository;
@Autowired
private VectorStoreRepository vectorStoreRepository;
public void deleteRecords(DeletionRequest deletionRequest) {
try {
boolean KSDocumentExists = deletionRequest.getKsDocumentId() != null && !deletionRequest.getKsDocumentId().isEmpty() && ksDocumentRepository.existsById(deletionRequest.getKsDocumentId());
boolean KSIngestionInfoExists = deletionRequest.getKsIngestionInfoId() != null && !deletionRequest.getKsIngestionInfoId().isEmpty() && ksIngestionInfoRepository.existsById(deletionRequest.getKsIngestionInfoId());
boolean vectorStoreExists = deletionRequest.getKsApplicationName() != null && deletionRequest.getKsDocSource() != null && deletionRequest.getKsFileSource() != null && deletionRequest.getKsDoctype() != null;
List<VectorStore> vectorStoreMetadataDetails = vectorStoreExists ? vectorStoreRepository.findByMetadata(deletionRequest.getKsDoctype(), deletionRequest.getKsDocSource(), deletionRequest.getKsFileSource(), deletionRequest.getKsApplicationName()) : List.of();
if (KSDocumentExists && KSIngestionInfoExists && !vectorStoreMetadataDetails.isEmpty()) {
if (deletionRequest.getKsDocumentId() != null && !deletionRequest.getKsDocumentId().isEmpty()) {
ksDocumentRepository.deleteById(deletionRequest.getKsDocumentId());
logger.info("KSDocument with id {} deleted successfully.", deletionRequest.getKsDocumentId());
}
if (deletionRequest.getKsIngestionInfoId() != null && !deletionRequest.getKsIngestionInfoId().isEmpty()) {
ksIngestionInfoRepository.deleteById(deletionRequest.getKsIngestionInfoId());
logger.info("KSIngestionInfo with id {} deleted successfully.", deletionRequest.getKsIngestionInfoId());
}
for (VectorStore store : vectorStoreMetadataDetails) {
vectorStoreRepository.deleteById(store.getId());
logger.info("VectorStore with id {} deleted successfully.", store.getId());
}
logger.info("All records deleted successfully.");
} else {
if (!KSDocumentExists) {
logger.warn("KSDocument with id {} does not exist.", deletionRequest.getKsDocumentId());
} else if (!KSIngestionInfoExists) {
logger.warn("KSIngestionInfo with id {} does not exist.", deletionRequest.getKsIngestionInfoId());
} else if (vectorStoreMetadataDetails.isEmpty()) {
logger.warn("No VectorStore Data available");
}
}
} catch (Exception e) {
logger.error("An error occurred while deleting records: ", e);
throw new RuntimeException("An error occurred while deleting records", e);
}
}
}

4
src/main/java/com/olympus/apollo/services/KSIngestor.java
View File

@@ -174,12 +174,12 @@ public class KSIngestor {
}
}
public List<String> testSimilaritySearch(String query,String filter_doc_type) {
public List<String> testSimilaritySearch(String query,String filterQuery) {
List<Document> docs = vectorStore.similaritySearch(
SearchRequest.defaults()
.withQuery(query)
.withTopK(5).withSimilarityThreshold(0.8)
.withFilterExpression("'KsDoctype'=='"+filter_doc_type+"'"));
.withFilterExpression(filterQuery));
List<String> result = new ArrayList<String>();
for (Document doc : docs) {

4
src/main/resources/application.properties
View File

@@ -24,6 +24,6 @@ spring.ai.vectorstore.mongodb.initialize-schema=false
spring.ai.openai.api-key=sk-proj-j3TFJ0h348DIzMrYYfyUT3BlbkFJjk4HMc8A2ux2Asg8Y7H1
#size filter
spring.servlet.multipart.max-file-size=10MB
spring.servlet.multipart.max-request-size=10MB
spring.servlet.multipart.max-file-size=100MB
spring.servlet.multipart.max-request-size=100MB