admin/apollo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

deletion for all services

Browse Source
This commit is contained in:
sumedh
2024-08-14 20:02:51 +05:30
parent b4482aabee
commit 81fda6ee70
3 changed files with 72 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
src/main/java/com/olympus/apollo/services/DeletionService.java
View File

@@ -1,7 +1,9 @@
package com.olympus.apollo.services;
import com.olympus.apollo.dto.DeleteGitRepoDetailsRequest;
import com.olympus.apollo.models.KSGitInfo;
import com.olympus.apollo.models.KSGitIngestionInfo;
import com.olympus.apollo.models.KSIngestionInfo;
import com.olympus.apollo.repository.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -13,6 +15,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.Optional;
@Service
public class DeletionService {
@@ -40,6 +43,7 @@ public class DeletionService {
boolean KSIngestionInfoExists = deletionRequest.getKsIngestionInfoId() != null && !deletionRequest.getKsIngestionInfoId().isEmpty() && ksIngestionInfoRepository.existsById(deletionRequest.getKsIngestionInfoId());
boolean vectorStoreExists = deletionRequest.getKsApplicationName() != null && deletionRequest.getKsDocSource() != null && deletionRequest.getKsFileSource() != null && deletionRequest.getKsDoctype() != null;
List<VectorStore> vectorStoreMetadataDetails = vectorStoreExists ? vectorStoreRepository.findByMetadata(deletionRequest.getKsDoctype(), deletionRequest.getKsDocSource(), deletionRequest.getKsFileSource(), deletionRequest.getKsApplicationName()) : List.of();
if (KSDocumentExists && KSIngestionInfoExists && !vectorStoreMetadataDetails.isEmpty()) {
@@ -81,34 +85,67 @@ public class DeletionService {
boolean KSGitIngestionInfoExists = deleteGitRepoDetailsRequest.getKsGitIngestionInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitIngestionInfoId().isEmpty() && ksGitIngestionInfoRepository.existsById(deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
logger.info("KSGitIngestionInfo with id {} exists.", deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
boolean vectorStoreGitDetailsExists = deleteGitRepoDetailsRequest.getKsApplicationName() != null && deleteGitRepoDetailsRequest.getKsDocSource() != null && deleteGitRepoDetailsRequest.getKsFileSource() != null && deleteGitRepoDetailsRequest.getKsDoctype() != null;
logger.info("vectorStoreGitDetails records exists.");
Optional<KSGitInfo> ksGitInfo = ksGitInfoRepository.findById(deleteGitRepoDetailsRequest.getKsGitInfoId());
String ingestionStatus = ksGitInfo.get().getIngestionStatus();
logger.info("Ingestion Status is {}.",ingestionStatus);
List<VectorStore> vectorStoreMetadataDetails = vectorStoreGitDetailsExists ? vectorStoreRepository.findByMetadata(deleteGitRepoDetailsRequest.getKsDoctype(), deleteGitRepoDetailsRequest.getKsDocSource(), deleteGitRepoDetailsRequest.getKsFileSource(), deleteGitRepoDetailsRequest.getKsApplicationName()) : List.of();
if (KSGitInfoExists && KSGitIngestionInfoExists && !vectorStoreMetadataDetails.isEmpty()) {
if (deleteGitRepoDetailsRequest.getKsGitInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitInfoId().isEmpty()) {
ksGitInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitInfoId());
logger.info("KsGitInfo with id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsGitInfoId());
}
if (KSGitInfoExists && KSGitIngestionInfoExists) {
if(ingestionStatus.equals("ERROR")){
if (deleteGitRepoDetailsRequest.getKsGitInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitInfoId().isEmpty()) {
ksGitInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitInfoId());
logger.info("KsGitInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitInfoId());
}
if (deleteGitRepoDetailsRequest.getKsGitIngestionInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitIngestionInfoId().isEmpty()) {
ksGitIngestionInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
logger.info("KSGitIngestionInfo with id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
}
if (deleteGitRepoDetailsRequest.getKsGitIngestionInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitIngestionInfoId().isEmpty()) {
ksGitIngestionInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
logger.info("KSGitIngestionInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
}
if( !vectorStoreMetadataDetails.isEmpty()){
for (VectorStore store : vectorStoreMetadataDetails) {
vectorStoreRepository.deleteById(store.getId());
logger.info("VectorStore with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), store.getId());
}
}
} else if (ingestionStatus.equals("INGESTED") && !vectorStoreMetadataDetails.isEmpty()) {
if (deleteGitRepoDetailsRequest.getKsGitInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitInfoId().isEmpty()) {
ksGitInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitInfoId());
logger.info("KsGitInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitInfoId());
}
if (deleteGitRepoDetailsRequest.getKsGitIngestionInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitIngestionInfoId().isEmpty()) {
ksGitIngestionInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
logger.info("KSGitIngestionInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
}
for (VectorStore store : vectorStoreMetadataDetails) {
vectorStoreRepository.deleteById(store.getId());
logger.info("VectorStore with id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), store.getId());
}
logger.info("All records deleted successfully.");
}else if (ingestionStatus.equals("NEW") && vectorStoreMetadataDetails.isEmpty()) {
if (deleteGitRepoDetailsRequest.getKsGitInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitInfoId().isEmpty()) {
ksGitInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitInfoId());
logger.info("KsGitInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitInfoId());
}
if (deleteGitRepoDetailsRequest.getKsGitIngestionInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitIngestionInfoId().isEmpty()) {
ksGitIngestionInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
logger.info("KSGitIngestionInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
}
logger.info("records deleted successfully.");
}
for (VectorStore store : vectorStoreMetadataDetails) {
vectorStoreRepository.deleteById(store.getId());
logger.info("VectorStore with id {} deleted successfully.", store.getId());
}
logger.info("All records deleted successfully.");
} else {
if (!KSGitInfoExists) {
logger.warn("getKsGitInfo with id {} does not exist.", deleteGitRepoDetailsRequest.getKsGitInfoId());
logger.warn("getKsGitInfo with reponame {} and id {} does not exist.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitInfoId());
} else if (!KSGitIngestionInfoExists) {
logger.warn("KSGitIngestionInfo with id {} does not exist.", deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
logger.warn("KSGitIngestionInfo with reponame {} and id {} does not exist.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
} else if (vectorStoreMetadataDetails.isEmpty()) {
logger.warn("No VectorStore Data available");
}
}
} catch (Exception e) {

15
src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java
View File

@@ -33,7 +33,6 @@ import com.olympus.apollo.repository.KSGitInfoRepository;
@Service
public class GitRepositoryIngestor {
@Value("${ingestion.repository.basepath}")
private String ingestionRepositoryBasePath;
@@ -50,14 +49,15 @@ public class GitRepositoryIngestor {
@Async
public CompletableFuture<Void> ingestGitRepository(String repo) {
String repoPath = ingestionRepositoryBasePath+"\\" + repo + "\\";
logger.info("Repository path : " + repoPath);
//String repoPath = "C:\\Users\\s.shamrao.shinde\\GenAIStorage\\" + repo + "\\";
//String repoPath = ksGitInfo.getRepoPath() + repo + "\\";
//logger.info("Repository path : " + repoPath);
Optional<KSGitInfo> optionalDocument = ksGitInfoRepository.findByRepoName(repo);
if (optionalDocument.isPresent()) {
KSGitInfo ksGitInfo = optionalDocument.get();
if ("NEW".equals(ksGitInfo.getIngestionStatus())) {
ingestRepo(repoPath, ksGitInfo);
if ("NEW".equals(ksGitInfo.getIngestionStatus()) || "ERROR".equals(ksGitInfo.getIngestionStatus())) {
ingestRepo(repo, ksGitInfo);
} else {
logger.info("OOPS: Document is already Injected");
}
@@ -67,7 +67,10 @@ public class GitRepositoryIngestor {
return CompletableFuture.completedFuture(null);
}
private void ingestRepo(String repoPath, KSGitInfo ksGitInfo) {
private void ingestRepo(String repo, KSGitInfo ksGitInfo) {
String repoPath = ingestionRepositoryBasePath +"\\"+ repo + "\\";
logger.info("Repository path : " + repoPath);
try (Git git = Git.open(new File(repoPath))) {
ksGitInfo.setIngestionStatus("IN PROGRESS");

6
src/main/java/com/olympus/apollo/services/StorageProperties.java
View File

@@ -1,13 +1,17 @@
package com.olympus.apollo.services;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.beans.factory.annotation.Value;
@ConfigurationProperties("storage")
public class StorageProperties {
/**
* Folder location for storing files
*/
private String location = "C:\\Users\\s.shamrao.shinde\\GenAIStorage";
@Value("${ingestion.repository.basepath}")
private String ingestionRepositoryBasePath;
private String location = ingestionRepositoryBasePath;
//private String location = "C:\\WindTre\\Repository";
public String getLocation() {
return location;