diff --git a/src/main/java/com/olympus/apollo/controllers/FeApi/VectorStoreController.java b/src/main/java/com/olympus/apollo/controllers/FeApi/VectorStoreController.java index 3b0e480..38a808c 100644 --- a/src/main/java/com/olympus/apollo/controllers/FeApi/VectorStoreController.java +++ b/src/main/java/com/olympus/apollo/controllers/FeApi/VectorStoreController.java @@ -1,13 +1,14 @@ package com.olympus.apollo.controllers.FeApi; +import com.olympus.apollo.dto.DeletionRequest; import com.olympus.apollo.dto.VectorStoreDetails; import com.olympus.apollo.models.VectorStore; import com.olympus.apollo.repository.VectorStoreRepository; +import com.olympus.apollo.services.DeletionService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; import java.util.List; -import java.util.stream.Collectors; @RestController @RequestMapping("/fe-api/vector-store") @@ -16,6 +17,9 @@ public class VectorStoreController { @Autowired private VectorStoreRepository vectorStoreRepository; + @Autowired + private DeletionService deletionService; + @GetMapping("") public List getAllVectorStore(){ List result = (List) vectorStoreRepository.findAll(); @@ -39,4 +43,10 @@ public class VectorStoreController { return vectorStoreRepository.findAllDetails(); } + @PostMapping("/deleteRecords") + public String deleteRecords(@RequestBody DeletionRequest deletionRequest){ + deletionService.deleteRecords(deletionRequest); + return "Records Deleted Successfully"; + } + } diff --git a/src/main/java/com/olympus/apollo/controllers/KSFileController.java b/src/main/java/com/olympus/apollo/controllers/KSFileController.java index 65d4f85..54c7520 100644 --- a/src/main/java/com/olympus/apollo/controllers/KSFileController.java +++ b/src/main/java/com/olympus/apollo/controllers/KSFileController.java @@ -53,7 +53,7 @@ public class KSFileController { metadata.put("KsApplicationName", fileUploadDTO.getKsApplicationName()); metadata.put("KsDoctype", fileUploadDTO.getKsDocType()); metadata.put("KsDocSource", fileUploadDTO.getKsDocSource()); - metadata.put("Source", file.getOriginalFilename()); + metadata.put("KsFileSource", file.getOriginalFilename()); ksIngestionInfo.setMetadata(metadata); ksIngestionInfo.setDefaultChunkSize(fileUploadDTO.getDefaultChunkSize()); diff --git a/src/main/java/com/olympus/apollo/controllers/TestController.java b/src/main/java/com/olympus/apollo/controllers/TestController.java index 761aabd..fb55a50 100644 --- a/src/main/java/com/olympus/apollo/controllers/TestController.java +++ b/src/main/java/com/olympus/apollo/controllers/TestController.java @@ -34,8 +34,8 @@ public class TestController { } @GetMapping("test/query_vector") - public List testSimilaritySearch(@RequestParam String query, @RequestParam String type) { - return ksIngestor.testSimilaritySearch(query,type); + public List testSimilaritySearch(@RequestParam String query, @RequestParam String filterQuery) { + return ksIngestor.testSimilaritySearch(query,filterQuery); } @GetMapping("test/delete") diff --git a/src/main/java/com/olympus/apollo/dto/DeletionRequest.java b/src/main/java/com/olympus/apollo/dto/DeletionRequest.java new file mode 100644 index 0000000..034d6d5 --- /dev/null +++ b/src/main/java/com/olympus/apollo/dto/DeletionRequest.java @@ -0,0 +1,15 @@ +package com.olympus.apollo.dto; + +import lombok.Getter; +import lombok.Setter; + +@Getter @Setter +public class DeletionRequest { + private String ksDocumentId; + private String ksIngestionInfoId; + private String ksDoctype; + private String ksDocSource; + private String ksFileSource; + private String ksApplicationName; +} + diff --git a/src/main/java/com/olympus/apollo/dto/VectorStoreMetadataDetails.java b/src/main/java/com/olympus/apollo/dto/VectorStoreMetadataDetails.java index e891af5..8a7cd41 100644 --- a/src/main/java/com/olympus/apollo/dto/VectorStoreMetadataDetails.java +++ b/src/main/java/com/olympus/apollo/dto/VectorStoreMetadataDetails.java @@ -9,7 +9,6 @@ public class VectorStoreMetadataDetails { private String KsApplicationName; private String KsDoctype; private String KsDocSource; - private String source; - private String Source; - + private String KsFileSource; } + diff --git a/src/main/java/com/olympus/apollo/repository/VectorStoreRepository.java b/src/main/java/com/olympus/apollo/repository/VectorStoreRepository.java index 33ee6cb..968437a 100644 --- a/src/main/java/com/olympus/apollo/repository/VectorStoreRepository.java +++ b/src/main/java/com/olympus/apollo/repository/VectorStoreRepository.java @@ -1,10 +1,10 @@ package com.olympus.apollo.repository; import com.olympus.apollo.dto.VectorStoreDetails; -import com.olympus.apollo.dto.VectorStoreMetadataDetails; import com.olympus.apollo.models.VectorStore; import org.springframework.data.mongodb.repository.Aggregation; import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.data.mongodb.repository.Query; import org.springframework.stereotype.Repository; import java.util.List; @@ -21,4 +21,7 @@ public interface VectorStoreRepository extends MongoRepository findAllDetails(); + + @Query("{'metadata.KsDoctype': ?0, 'metadata.KsDocSource': ?1, 'metadata.KsFileSource': ?2, 'metadata.KsApplicationName': ?3}") + List findByMetadata(String ksDoctype, String ksDocSource, String ksFileSource, String ksApplicationName); } diff --git a/src/main/java/com/olympus/apollo/services/DeletionService.java b/src/main/java/com/olympus/apollo/services/DeletionService.java new file mode 100644 index 0000000..650c666 --- /dev/null +++ b/src/main/java/com/olympus/apollo/services/DeletionService.java @@ -0,0 +1,70 @@ +package com.olympus.apollo.services; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.olympus.apollo.dto.DeletionRequest; +import com.olympus.apollo.dto.VectorStoreMetadataDetails; +import com.olympus.apollo.models.VectorStore; +import com.olympus.apollo.repository.KSDocumentRepository; +import com.olympus.apollo.repository.KSIngestionInfoRepository; +import com.olympus.apollo.repository.VectorStoreRepository; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +public class DeletionService { + + private static final Logger logger = LoggerFactory.getLogger(DeletionService.class); + + @Autowired + private KSDocumentRepository ksDocumentRepository; + + @Autowired + private KSIngestionInfoRepository ksIngestionInfoRepository; + + @Autowired + private VectorStoreRepository vectorStoreRepository; + + public void deleteRecords(DeletionRequest deletionRequest) { + try { + boolean KSDocumentExists = deletionRequest.getKsDocumentId() != null && !deletionRequest.getKsDocumentId().isEmpty() && ksDocumentRepository.existsById(deletionRequest.getKsDocumentId()); + boolean KSIngestionInfoExists = deletionRequest.getKsIngestionInfoId() != null && !deletionRequest.getKsIngestionInfoId().isEmpty() && ksIngestionInfoRepository.existsById(deletionRequest.getKsIngestionInfoId()); + boolean vectorStoreExists = deletionRequest.getKsApplicationName() != null && deletionRequest.getKsDocSource() != null && deletionRequest.getKsFileSource() != null && deletionRequest.getKsDoctype() != null; + + List vectorStoreMetadataDetails = vectorStoreExists ? vectorStoreRepository.findByMetadata(deletionRequest.getKsDoctype(), deletionRequest.getKsDocSource(), deletionRequest.getKsFileSource(), deletionRequest.getKsApplicationName()) : List.of(); + + if (KSDocumentExists && KSIngestionInfoExists && !vectorStoreMetadataDetails.isEmpty()) { + if (deletionRequest.getKsDocumentId() != null && !deletionRequest.getKsDocumentId().isEmpty()) { + ksDocumentRepository.deleteById(deletionRequest.getKsDocumentId()); + logger.info("KSDocument with id {} deleted successfully.", deletionRequest.getKsDocumentId()); + } + + if (deletionRequest.getKsIngestionInfoId() != null && !deletionRequest.getKsIngestionInfoId().isEmpty()) { + ksIngestionInfoRepository.deleteById(deletionRequest.getKsIngestionInfoId()); + logger.info("KSIngestionInfo with id {} deleted successfully.", deletionRequest.getKsIngestionInfoId()); + } + + for (VectorStore store : vectorStoreMetadataDetails) { + vectorStoreRepository.deleteById(store.getId()); + logger.info("VectorStore with id {} deleted successfully.", store.getId()); + } + logger.info("All records deleted successfully."); + } else { + if (!KSDocumentExists) { + logger.warn("KSDocument with id {} does not exist.", deletionRequest.getKsDocumentId()); + } else if (!KSIngestionInfoExists) { + logger.warn("KSIngestionInfo with id {} does not exist.", deletionRequest.getKsIngestionInfoId()); + } else if (vectorStoreMetadataDetails.isEmpty()) { + logger.warn("No VectorStore Data available"); + + } + } + } catch (Exception e) { + logger.error("An error occurred while deleting records: ", e); + throw new RuntimeException("An error occurred while deleting records", e); + } + } +} diff --git a/src/main/java/com/olympus/apollo/services/KSIngestor.java b/src/main/java/com/olympus/apollo/services/KSIngestor.java index 276932e..a692e4b 100644 --- a/src/main/java/com/olympus/apollo/services/KSIngestor.java +++ b/src/main/java/com/olympus/apollo/services/KSIngestor.java @@ -174,12 +174,12 @@ public class KSIngestor { } } - public List testSimilaritySearch(String query,String filter_doc_type) { + public List testSimilaritySearch(String query,String filterQuery) { List docs = vectorStore.similaritySearch( SearchRequest.defaults() .withQuery(query) .withTopK(5).withSimilarityThreshold(0.8) - .withFilterExpression("'KsDoctype'=='"+filter_doc_type+"'")); + .withFilterExpression(filterQuery)); List result = new ArrayList(); for (Document doc : docs) { diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index b9f6efe..121bc5b 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -24,6 +24,6 @@ spring.ai.vectorstore.mongodb.initialize-schema=false spring.ai.openai.api-key=sk-proj-j3TFJ0h348DIzMrYYfyUT3BlbkFJjk4HMc8A2ux2Asg8Y7H1 #size filter -spring.servlet.multipart.max-file-size=10MB -spring.servlet.multipart.max-request-size=10MB +spring.servlet.multipart.max-file-size=100MB +spring.servlet.multipart.max-request-size=100MB