diff --git a/src/main/java/com/olympus/apollo/services/DeletionService.java b/src/main/java/com/olympus/apollo/services/DeletionService.java index 58ae2ed..467712b 100644 --- a/src/main/java/com/olympus/apollo/services/DeletionService.java +++ b/src/main/java/com/olympus/apollo/services/DeletionService.java @@ -1,7 +1,9 @@ package com.olympus.apollo.services; import com.olympus.apollo.dto.DeleteGitRepoDetailsRequest; +import com.olympus.apollo.models.KSGitInfo; import com.olympus.apollo.models.KSGitIngestionInfo; +import com.olympus.apollo.models.KSIngestionInfo; import com.olympus.apollo.repository.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -13,6 +15,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.util.List; +import java.util.Optional; @Service public class DeletionService { @@ -40,6 +43,7 @@ public class DeletionService { boolean KSIngestionInfoExists = deletionRequest.getKsIngestionInfoId() != null && !deletionRequest.getKsIngestionInfoId().isEmpty() && ksIngestionInfoRepository.existsById(deletionRequest.getKsIngestionInfoId()); boolean vectorStoreExists = deletionRequest.getKsApplicationName() != null && deletionRequest.getKsDocSource() != null && deletionRequest.getKsFileSource() != null && deletionRequest.getKsDoctype() != null; + List vectorStoreMetadataDetails = vectorStoreExists ? vectorStoreRepository.findByMetadata(deletionRequest.getKsDoctype(), deletionRequest.getKsDocSource(), deletionRequest.getKsFileSource(), deletionRequest.getKsApplicationName()) : List.of(); if (KSDocumentExists && KSIngestionInfoExists && !vectorStoreMetadataDetails.isEmpty()) { @@ -81,34 +85,67 @@ public class DeletionService { boolean KSGitIngestionInfoExists = deleteGitRepoDetailsRequest.getKsGitIngestionInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitIngestionInfoId().isEmpty() && ksGitIngestionInfoRepository.existsById(deleteGitRepoDetailsRequest.getKsGitIngestionInfoId()); logger.info("KSGitIngestionInfo with id {} exists.", deleteGitRepoDetailsRequest.getKsGitIngestionInfoId()); boolean vectorStoreGitDetailsExists = deleteGitRepoDetailsRequest.getKsApplicationName() != null && deleteGitRepoDetailsRequest.getKsDocSource() != null && deleteGitRepoDetailsRequest.getKsFileSource() != null && deleteGitRepoDetailsRequest.getKsDoctype() != null; - logger.info("vectorStoreGitDetails records exists."); + + Optional ksGitInfo = ksGitInfoRepository.findById(deleteGitRepoDetailsRequest.getKsGitInfoId()); + String ingestionStatus = ksGitInfo.get().getIngestionStatus(); + logger.info("Ingestion Status is {}.",ingestionStatus); List vectorStoreMetadataDetails = vectorStoreGitDetailsExists ? vectorStoreRepository.findByMetadata(deleteGitRepoDetailsRequest.getKsDoctype(), deleteGitRepoDetailsRequest.getKsDocSource(), deleteGitRepoDetailsRequest.getKsFileSource(), deleteGitRepoDetailsRequest.getKsApplicationName()) : List.of(); - if (KSGitInfoExists && KSGitIngestionInfoExists && !vectorStoreMetadataDetails.isEmpty()) { - if (deleteGitRepoDetailsRequest.getKsGitInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitInfoId().isEmpty()) { - ksGitInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitInfoId()); - logger.info("KsGitInfo with id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsGitInfoId()); - } + if (KSGitInfoExists && KSGitIngestionInfoExists) { + if(ingestionStatus.equals("ERROR")){ + if (deleteGitRepoDetailsRequest.getKsGitInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitInfoId().isEmpty()) { + ksGitInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitInfoId()); + logger.info("KsGitInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitInfoId()); + } - if (deleteGitRepoDetailsRequest.getKsGitIngestionInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitIngestionInfoId().isEmpty()) { - ksGitIngestionInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitIngestionInfoId()); - logger.info("KSGitIngestionInfo with id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsGitIngestionInfoId()); - } + if (deleteGitRepoDetailsRequest.getKsGitIngestionInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitIngestionInfoId().isEmpty()) { + ksGitIngestionInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitIngestionInfoId()); + logger.info("KSGitIngestionInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitIngestionInfoId()); + } + if( !vectorStoreMetadataDetails.isEmpty()){ + for (VectorStore store : vectorStoreMetadataDetails) { + vectorStoreRepository.deleteById(store.getId()); + logger.info("VectorStore with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), store.getId()); + } + } + + } else if (ingestionStatus.equals("INGESTED") && !vectorStoreMetadataDetails.isEmpty()) { + if (deleteGitRepoDetailsRequest.getKsGitInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitInfoId().isEmpty()) { + ksGitInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitInfoId()); + logger.info("KsGitInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitInfoId()); + } + + if (deleteGitRepoDetailsRequest.getKsGitIngestionInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitIngestionInfoId().isEmpty()) { + ksGitIngestionInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitIngestionInfoId()); + logger.info("KSGitIngestionInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitIngestionInfoId()); + } + + for (VectorStore store : vectorStoreMetadataDetails) { + vectorStoreRepository.deleteById(store.getId()); + logger.info("VectorStore with id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), store.getId()); + } + logger.info("All records deleted successfully."); + }else if (ingestionStatus.equals("NEW") && vectorStoreMetadataDetails.isEmpty()) { + if (deleteGitRepoDetailsRequest.getKsGitInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitInfoId().isEmpty()) { + ksGitInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitInfoId()); + logger.info("KsGitInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitInfoId()); + } + + if (deleteGitRepoDetailsRequest.getKsGitIngestionInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitIngestionInfoId().isEmpty()) { + ksGitIngestionInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitIngestionInfoId()); + logger.info("KSGitIngestionInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitIngestionInfoId()); + } + logger.info("records deleted successfully."); + } - for (VectorStore store : vectorStoreMetadataDetails) { - vectorStoreRepository.deleteById(store.getId()); - logger.info("VectorStore with id {} deleted successfully.", store.getId()); - } - logger.info("All records deleted successfully."); } else { if (!KSGitInfoExists) { - logger.warn("getKsGitInfo with id {} does not exist.", deleteGitRepoDetailsRequest.getKsGitInfoId()); + logger.warn("getKsGitInfo with reponame {} and id {} does not exist.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitInfoId()); } else if (!KSGitIngestionInfoExists) { - logger.warn("KSGitIngestionInfo with id {} does not exist.", deleteGitRepoDetailsRequest.getKsGitIngestionInfoId()); + logger.warn("KSGitIngestionInfo with reponame {} and id {} does not exist.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitIngestionInfoId()); } else if (vectorStoreMetadataDetails.isEmpty()) { logger.warn("No VectorStore Data available"); - } } } catch (Exception e) { @@ -116,4 +153,4 @@ public class DeletionService { throw new RuntimeException("An error occurred while deleting records", e); } } -} +} \ No newline at end of file diff --git a/src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java b/src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java index 7a60287..9ab6dea 100644 --- a/src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java +++ b/src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java @@ -33,7 +33,6 @@ import com.olympus.apollo.repository.KSGitInfoRepository; @Service public class GitRepositoryIngestor { - @Value("${ingestion.repository.basepath}") private String ingestionRepositoryBasePath; @@ -50,14 +49,15 @@ public class GitRepositoryIngestor { @Async public CompletableFuture ingestGitRepository(String repo) { - String repoPath = ingestionRepositoryBasePath+"\\" + repo + "\\"; - logger.info("Repository path : " + repoPath); + //String repoPath = "C:\\Users\\s.shamrao.shinde\\GenAIStorage\\" + repo + "\\"; + //String repoPath = ksGitInfo.getRepoPath() + repo + "\\"; + //logger.info("Repository path : " + repoPath); Optional optionalDocument = ksGitInfoRepository.findByRepoName(repo); if (optionalDocument.isPresent()) { KSGitInfo ksGitInfo = optionalDocument.get(); - if ("NEW".equals(ksGitInfo.getIngestionStatus())) { - ingestRepo(repoPath, ksGitInfo); + if ("NEW".equals(ksGitInfo.getIngestionStatus()) || "ERROR".equals(ksGitInfo.getIngestionStatus())) { + ingestRepo(repo, ksGitInfo); } else { logger.info("OOPS: Document is already Injected"); } @@ -67,7 +67,10 @@ public class GitRepositoryIngestor { return CompletableFuture.completedFuture(null); } - private void ingestRepo(String repoPath, KSGitInfo ksGitInfo) { + + private void ingestRepo(String repo, KSGitInfo ksGitInfo) { + String repoPath = ingestionRepositoryBasePath +"\\"+ repo + "\\"; + logger.info("Repository path : " + repoPath); try (Git git = Git.open(new File(repoPath))) { ksGitInfo.setIngestionStatus("IN PROGRESS"); @@ -399,4 +402,4 @@ public class GitRepositoryIngestor { return metadata; } -} +} \ No newline at end of file diff --git a/src/main/java/com/olympus/apollo/services/StorageProperties.java b/src/main/java/com/olympus/apollo/services/StorageProperties.java index a9e10cd..05bb270 100644 --- a/src/main/java/com/olympus/apollo/services/StorageProperties.java +++ b/src/main/java/com/olympus/apollo/services/StorageProperties.java @@ -1,13 +1,17 @@ package com.olympus.apollo.services; import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.beans.factory.annotation.Value; @ConfigurationProperties("storage") public class StorageProperties { /** * Folder location for storing files */ - private String location = "C:\\Users\\s.shamrao.shinde\\GenAIStorage"; + @Value("${ingestion.repository.basepath}") + private String ingestionRepositoryBasePath; + + private String location = ingestionRepositoryBasePath; //private String location = "C:\\WindTre\\Repository"; public String getLocation() { return location; @@ -17,4 +21,4 @@ public class StorageProperties { this.location = location; } -} +} \ No newline at end of file