diff --git a/.gitignore b/.gitignore index d2d0fbd..a8d4013 100644 --- a/.gitignore +++ b/.gitignore @@ -33,8 +33,6 @@ build/ .vscode/ ##changes in filepath before deploy -src/main/java/com/olympus/apollo/services/StorageProperties.java -src/main/resources/application.properties -src/main/java/com/olympus/apollo/services/GitService.java -src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java +#src/main/java/com/olympus/apollo/services/StorageProperties.java +#src/main/resources/application.properties diff --git a/pom.xml b/pom.xml index 6f2a914..ab34a6d 100644 --- a/pom.xml +++ b/pom.xml @@ -37,6 +37,11 @@ spring-boot-starter-web + + org.springframework.boot + spring-boot-starter-websocket + + org.springframework.boot spring-boot-starter-data-mongodb diff --git a/src/main/java/com/olympus/apollo/ApolloApplication.java b/src/main/java/com/olympus/apollo/ApolloApplication.java index f0b54c5..d183ca7 100644 --- a/src/main/java/com/olympus/apollo/ApolloApplication.java +++ b/src/main/java/com/olympus/apollo/ApolloApplication.java @@ -5,7 +5,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.scheduling.annotation.EnableAsync; -import com.olympus.apollo.services.StorageProperties; +import com.olympus.apollo.properties.StorageProperties; @SpringBootApplication @EnableConfigurationProperties(StorageProperties.class) diff --git a/src/main/java/com/olympus/apollo/config/WebSocketConfig.java b/src/main/java/com/olympus/apollo/config/WebSocketConfig.java new file mode 100644 index 0000000..626e533 --- /dev/null +++ b/src/main/java/com/olympus/apollo/config/WebSocketConfig.java @@ -0,0 +1,25 @@ +package com.olympus.apollo.config; + +import org.springframework.context.annotation.Configuration; +import org.springframework.messaging.simp.config.MessageBrokerRegistry; +import org.springframework.web.socket.config.annotation.EnableWebSocketMessageBroker; +import org.springframework.web.socket.config.annotation.StompEndpointRegistry; +import org.springframework.web.socket.config.annotation.WebSocketMessageBrokerConfigurer; + +@Configuration +@EnableWebSocketMessageBroker +public class WebSocketConfig implements WebSocketMessageBrokerConfigurer { + + @Override + public void registerStompEndpoints(StompEndpointRegistry registry) { + registry.addEndpoint("/ws/endpoint") + .setAllowedOrigins("http://apollo.olympusai.live") + .withSockJS(); + } + + @Override + public void configureMessageBroker(MessageBrokerRegistry config) { + config.enableSimpleBroker("/topic"); //should be used as prefix while sending request from spring to browser + config.setApplicationDestinationPrefixes("/app"); //should be used as prefix from browser to spring server + } +} diff --git a/src/main/java/com/olympus/apollo/controllers/FeApi/KSGitController.java b/src/main/java/com/olympus/apollo/controllers/FeApi/KSGitController.java index 96f7b93..20888ab 100644 --- a/src/main/java/com/olympus/apollo/controllers/FeApi/KSGitController.java +++ b/src/main/java/com/olympus/apollo/controllers/FeApi/KSGitController.java @@ -5,12 +5,19 @@ import java.text.SimpleDateFormat; import java.util.Date; import java.util.HashMap; import java.util.List; +import java.util.concurrent.CompletableFuture; import com.olympus.apollo.dto.*; +import com.olympus.apollo.exception.GitCloneException; import com.olympus.apollo.services.GitService; +import com.olympus.apollo.utils.GitUtils; import org.eclipse.jgit.api.errors.GitAPIException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpStatus; +import org.springframework.http.HttpStatusCode; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; @@ -24,94 +31,79 @@ import com.olympus.apollo.services.KSGitInfoService; @RequestMapping("/fe-api/ks_git_repos") public class KSGitController { - @Autowired - private KSGitInfoRepository ksGitInfoRepository; - @Autowired - private KSGitIngestionInfoRepository ksGitIngestionInfoRepository; + @Autowired + private KSGitInfoRepository ksGitInfoRepository; + @Autowired + private KSGitIngestionInfoRepository ksGitIngestionInfoRepository; - @Autowired - private GitService gitService; + @Autowired + private GitService gitService; - @Value("${gitlab.path}") - private String basePath; + @Value("${gitlab.path}") + private String basePath; - @GetMapping("") - public List listGitInfo() { - List result = (List) ksGitInfoRepository.findAll(); - return result; - } + private static final Logger logger = LoggerFactory.getLogger(KSGitController.class); - @PostMapping("/uploadRepo") - public ResponseEntity handleGitUpload(@RequestBody KSGitUploadDTO ksGitUploadDTO) { + @GetMapping("") + public List listGitInfo() { + List result = (List) ksGitInfoRepository.findAll(); + return result; + } - KSGitInfo ksGitInfo = new KSGitInfo(); - ksGitInfo.setRepoName(ksGitUploadDTO.getRepoName()); - ksGitInfo.setBranch(ksGitUploadDTO.getBranch()); - ksGitInfo.setCommitId(ksGitUploadDTO.getCommitId()); - ksGitInfo.setRepoPath(ksGitUploadDTO.getRepoPath()); - ksGitInfo.setIngestionStatus("NEW"); - ksGitInfo.setIngestionDate(new Date()); - ksGitInfo.setIngestionDateFormat(new SimpleDateFormat("MM/dd/yy").format(new Date())); + @PostMapping("/uploadRepo") + public ResponseEntity handleGitUpload(@RequestBody KSGitUploadDTO ksGitUploadDTO) { - KSGitIngestionInfo ksGitIngestionInfo = new KSGitIngestionInfo(); - HashMap metadata = new HashMap<>(); - - metadata.put("KsApplicationName", ksGitUploadDTO.getRepoName()); - metadata.put("KsDoctype", "gitrepository"); - metadata.put("KsDocSource", "gitlab"); - metadata.put("KsFileSource", ksGitUploadDTO.getRepoName()); - - metadata.put("KsBranch", ksGitUploadDTO.getBranch()); - metadata.put("KsRepoName", ksGitUploadDTO.getRepoName()); + KSGitInfo ksGitInfo = new KSGitInfo(); + ksGitInfo.setRepoName(ksGitUploadDTO.getRepoName()); + ksGitInfo.setBranch(ksGitUploadDTO.getBranch()); + ksGitInfo.setCommitId(ksGitUploadDTO.getCommitId()); + ksGitInfo.setRepoPath(ksGitUploadDTO.getRepoPath()); + ksGitInfo.setIngestionStatus("NEW"); + ksGitInfo.setIngestionDate(new Date()); + ksGitInfo.setIngestionDateFormat(new SimpleDateFormat("MM/dd/yy").format(new Date())); - ksGitIngestionInfo.setMetadata(metadata); - ksGitIngestionInfo.setMinChunkSizeToEmbed(ksGitUploadDTO.getMinChunkSizeToEmbed()); - ksGitIngestionInfo.setMaxNumberOfChunks(ksGitUploadDTO.getMaxNumberOfChunks()); - ksGitIngestionInfo.setMinChunkSize(ksGitUploadDTO.getMinChunkSize()); - ksGitIngestionInfo.setDefaultChunkSize(ksGitUploadDTO.getDefaultChunkSize()); + KSGitIngestionInfo ksGitIngestionInfo = new KSGitIngestionInfo(); + HashMap metadata = new HashMap<>(); + metadata.put("KsApplicationName", ksGitUploadDTO.getRepoName()); + metadata.put("KsDoctype", "gitrepository"); + metadata.put("KsDocSource", "gitlab"); + metadata.put("KsFileSource", ksGitUploadDTO.getRepoName()); + + metadata.put("KsBranch", ksGitUploadDTO.getBranch()); + metadata.put("KsRepoName", ksGitUploadDTO.getRepoName()); + + ksGitIngestionInfo.setMetadata(metadata); + ksGitIngestionInfo.setMinChunkSizeToEmbed(ksGitUploadDTO.getMinChunkSizeToEmbed()); + ksGitIngestionInfo.setMaxNumberOfChunks(ksGitUploadDTO.getMaxNumberOfChunks()); + ksGitIngestionInfo.setMinChunkSize(ksGitUploadDTO.getMinChunkSize()); + ksGitIngestionInfo.setDefaultChunkSize(ksGitUploadDTO.getDefaultChunkSize()); + + ksGitIngestionInfoRepository.save(ksGitIngestionInfo); + ksGitInfo.setKsGitIngestionInfo(ksGitIngestionInfo); + ksGitInfoRepository.save(ksGitInfo); + + return ResponseEntity.ok("Upload successful"); + } + + //clone the repository + @PostMapping("/clone") + public ResponseEntity gitClone(@RequestBody GitCloneInput gitCloneInput) throws GitCloneException { + KSGitInfo ksGitInfo = GitUtils.createKSGitInfo(gitCloneInput, basePath); + KSGitIngestionInfo ksGitIngestionInfo = GitUtils.createKSGitIngestionInfo(gitCloneInput); ksGitIngestionInfoRepository.save(ksGitIngestionInfo); ksGitInfo.setKsGitIngestionInfo(ksGitIngestionInfo); ksGitInfoRepository.save(ksGitInfo); - return ResponseEntity.ok("Upload successful"); - } + // Start the async processing + gitService.cloneRepository(gitCloneInput.getSource(), gitCloneInput.getRepoName(), gitCloneInput.getBranch(), gitCloneInput.getGroup(), gitCloneInput.getTokenType(), ksGitInfo); - //clone the master branch from remote repository - @PostMapping("/clone") - public GitCloneOutput gitClone(@RequestBody GitCloneInput gitCloneInput) throws GitAPIException, IOException { - KSGitInfo ksGitInfo = new KSGitInfo(); - ksGitInfo.setRepoName(gitCloneInput.getRepoName()); - ksGitInfo.setBranch(gitCloneInput.getBranch()); - ksGitInfo.setCommitId(gitCloneInput.getCommitId()); - ksGitInfo.setRepoPath(basePath+"/"+gitCloneInput.getBranch()); - ksGitInfo.setIngestionStatus("NEW"); - ksGitInfo.setIngestionDate(new Date()); - ksGitInfo.setIngestionDateFormat(new SimpleDateFormat("MM/dd/yy").format(new Date())); - - KSGitIngestionInfo ksGitIngestionInfo = new KSGitIngestionInfo(); - HashMap metadata = new HashMap<>(); - - metadata.put("KsApplicationName", gitCloneInput.getRepoName()); - metadata.put("KsDoctype", "gitrepository"); - metadata.put("KsDocSource", "gitlab"); - metadata.put("KsFileSource", gitCloneInput.getRepoName()); - - metadata.put("KsBranch", gitCloneInput.getBranch()); - metadata.put("KsRepoName", gitCloneInput.getRepoName()); - - ksGitIngestionInfo.setMetadata(metadata); - ksGitIngestionInfo.setMinChunkSizeToEmbed(gitCloneInput.getMinChunkSizeToEmbed()); - ksGitIngestionInfo.setMaxNumberOfChunks(gitCloneInput.getMaxNumberOfChunks()); - ksGitIngestionInfo.setMinChunkSize(gitCloneInput.getMinChunkSize()); - ksGitIngestionInfo.setDefaultChunkSize(gitCloneInput.getDefaultChunkSize()); - - ksGitIngestionInfoRepository.save(ksGitIngestionInfo); - ksGitInfo.setKsGitIngestionInfo(ksGitIngestionInfo); - ksGitInfoRepository.save(ksGitInfo); - - return gitService.cloneRepository(gitCloneInput.getSource(),gitCloneInput.getRepoName(),gitCloneInput.getBranch(),gitCloneInput.getGroup(),gitCloneInput.getTokenType()); - } + //Return an immediate response + ResultDTO response = new ResultDTO(); + response.setMessage("Ingestion process initiated for repo: " + gitCloneInput.getRepoName() + " branch: " + gitCloneInput.getBranch()); + response.setSuccess(true); + return ResponseEntity.ok(response); + } /* curl --location 'http://localhost:8082/fe-api/ks_git_repos/clone' \ --header 'Content-Type: application/json' \ @@ -120,10 +112,10 @@ public class KSGitController { ' */ - //pull latest changes from master branch - @GetMapping("/pullchanges") - public GitPullOutput gitPull(@RequestParam String repoName,@RequestParam String branchName){ - return gitService.pullChanges(repoName,branchName); - } + //pull latest changes from master branch + @GetMapping("/pullchanges") + public GitPullOutput gitPull(@RequestParam String repoName, @RequestParam String branchName) { + return gitService.pullChanges(repoName, branchName); + } } diff --git a/src/main/java/com/olympus/apollo/controllers/KSFileController.java b/src/main/java/com/olympus/apollo/controllers/KSFileController.java index 308a6a0..a4b46bd 100644 --- a/src/main/java/com/olympus/apollo/controllers/KSFileController.java +++ b/src/main/java/com/olympus/apollo/controllers/KSFileController.java @@ -13,7 +13,7 @@ import com.olympus.apollo.models.KSDocument; import com.olympus.apollo.models.KSIngestionInfo; import com.olympus.apollo.repository.KSDocumentRepository; import com.olympus.apollo.repository.KSIngestionInfoRepository; -import com.olympus.apollo.services.StorageFileNotFoundException; +import com.olympus.apollo.exception.StorageFileNotFoundException; import com.olympus.apollo.services.StorageService; import com.olympus.apollo.dto.FileUploadDTO; diff --git a/src/main/java/com/olympus/apollo/controllers/TestController.java b/src/main/java/com/olympus/apollo/controllers/TestController.java index 6415512..8bbd325 100644 --- a/src/main/java/com/olympus/apollo/controllers/TestController.java +++ b/src/main/java/com/olympus/apollo/controllers/TestController.java @@ -1,7 +1,9 @@ package com.olympus.apollo.controllers; import java.util.List; +import java.util.concurrent.CompletableFuture; +import com.olympus.apollo.dto.ResultDTO; import com.olympus.apollo.services.GitService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,17 +57,15 @@ public class TestController { } @GetMapping("test/ingest_repo") - public ResponseEntity ingestRepo(@RequestParam String repoName,@RequestParam String branchName) { - try { - gitRepositoryIngestor.ingestGitRepository(repoName,branchName); - return ResponseEntity.ok("Ingestion Started"); - } catch (Exception e) { + public ResponseEntity ingestRepo(@RequestParam String repoName, @RequestParam String branchName) { + // Start the async processing + gitRepositoryIngestor.ingestGitRepository(repoName,branchName); - logger.error("Error during ingestion start", e); - - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) - .body("Error starting ingestion: " + e.getMessage()); - } + //Return an immediate response + ResultDTO response = new ResultDTO(); + response.setMessage("Ingestion process initiated for repo: " + repoName + " branch: " + branchName); + response.setSuccess(true); + return ResponseEntity.ok(response); } @GetMapping("test/reingest_repo") diff --git a/src/main/java/com/olympus/apollo/dto/ResultDTO.java b/src/main/java/com/olympus/apollo/dto/ResultDTO.java new file mode 100644 index 0000000..75864e0 --- /dev/null +++ b/src/main/java/com/olympus/apollo/dto/ResultDTO.java @@ -0,0 +1,10 @@ +package com.olympus.apollo.dto; + +import lombok.Getter; +import lombok.Setter; + +@Setter @Getter +public class ResultDTO { + private boolean success; + private String message; +} diff --git a/src/main/java/com/olympus/apollo/exception/BranchCheckoutException.java b/src/main/java/com/olympus/apollo/exception/BranchCheckoutException.java new file mode 100644 index 0000000..76a1371 --- /dev/null +++ b/src/main/java/com/olympus/apollo/exception/BranchCheckoutException.java @@ -0,0 +1,11 @@ +package com.olympus.apollo.exception; + +public class BranchCheckoutException extends Exception{ + public BranchCheckoutException(String message){ + super(message); + } + + public BranchCheckoutException(String message,Throwable cause){ + super(message,cause); + } +} diff --git a/src/main/java/com/olympus/apollo/exception/GitCloneException.java b/src/main/java/com/olympus/apollo/exception/GitCloneException.java new file mode 100644 index 0000000..f9ce65b --- /dev/null +++ b/src/main/java/com/olympus/apollo/exception/GitCloneException.java @@ -0,0 +1,11 @@ +package com.olympus.apollo.exception; + +public class GitCloneException extends Exception{ + public GitCloneException(String message){ + super(message); + } + + public GitCloneException(String message,Throwable cause){ + super(message,cause); + } +} diff --git a/src/main/java/com/olympus/apollo/exception/GlobalExceptionHandler.java b/src/main/java/com/olympus/apollo/exception/GlobalExceptionHandler.java new file mode 100644 index 0000000..5c57246 --- /dev/null +++ b/src/main/java/com/olympus/apollo/exception/GlobalExceptionHandler.java @@ -0,0 +1,25 @@ +package com.olympus.apollo.exception; + +import com.olympus.apollo.dto.GitCloneOutput; +import org.slf4j.LoggerFactory; +import org.slf4j.Logger; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.ResponseBody; + +@ControllerAdvice +public class GlobalExceptionHandler { + private static final Logger logger = LoggerFactory.getLogger(GlobalExceptionHandler.class); + + @ExceptionHandler(GitCloneException.class) + @ResponseBody + public ResponseEntity handleGitCloneException(GitCloneException ex) { + logger.error("Git clone exception: {}", ex.getMessage(), ex); + GitCloneOutput output = new GitCloneOutput(); + output.setMessage("Git clone error: " + ex.getMessage()); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(output); + } + +} diff --git a/src/main/java/com/olympus/apollo/services/StorageException.java b/src/main/java/com/olympus/apollo/exception/StorageException.java similarity index 85% rename from src/main/java/com/olympus/apollo/services/StorageException.java rename to src/main/java/com/olympus/apollo/exception/StorageException.java index b872b98..981a2d5 100644 --- a/src/main/java/com/olympus/apollo/services/StorageException.java +++ b/src/main/java/com/olympus/apollo/exception/StorageException.java @@ -1,4 +1,4 @@ -package com.olympus.apollo.services; +package com.olympus.apollo.exception; public class StorageException extends RuntimeException { public StorageException(String message) { diff --git a/src/main/java/com/olympus/apollo/services/StorageFileNotFoundException.java b/src/main/java/com/olympus/apollo/exception/StorageFileNotFoundException.java similarity index 86% rename from src/main/java/com/olympus/apollo/services/StorageFileNotFoundException.java rename to src/main/java/com/olympus/apollo/exception/StorageFileNotFoundException.java index 967e71b..317b3c9 100644 --- a/src/main/java/com/olympus/apollo/services/StorageFileNotFoundException.java +++ b/src/main/java/com/olympus/apollo/exception/StorageFileNotFoundException.java @@ -1,4 +1,4 @@ -package com.olympus.apollo.services; +package com.olympus.apollo.exception; public class StorageFileNotFoundException extends StorageException { diff --git a/src/main/java/com/olympus/apollo/exception/vectorStoreMetaDetailsEmptyException.java b/src/main/java/com/olympus/apollo/exception/vectorStoreMetaDetailsEmptyException.java new file mode 100644 index 0000000..05d5bc9 --- /dev/null +++ b/src/main/java/com/olympus/apollo/exception/vectorStoreMetaDetailsEmptyException.java @@ -0,0 +1,10 @@ +package com.olympus.apollo.exception; + +public class vectorStoreMetaDetailsEmptyException extends RuntimeException{ + public vectorStoreMetaDetailsEmptyException(String message){ + super(message); + } + public vectorStoreMetaDetailsEmptyException(String message,Throwable cause){ + super(message,cause); + } +} diff --git a/src/main/java/com/olympus/apollo/services/StorageProperties.java b/src/main/java/com/olympus/apollo/properties/StorageProperties.java similarity index 84% rename from src/main/java/com/olympus/apollo/services/StorageProperties.java rename to src/main/java/com/olympus/apollo/properties/StorageProperties.java index 3bc1c71..777b468 100644 --- a/src/main/java/com/olympus/apollo/services/StorageProperties.java +++ b/src/main/java/com/olympus/apollo/properties/StorageProperties.java @@ -1,7 +1,6 @@ -package com.olympus.apollo.services; +package com.olympus.apollo.properties; import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.beans.factory.annotation.Value; @ConfigurationProperties("storage") public class StorageProperties { diff --git a/src/main/java/com/olympus/apollo/services/DeletionService.java b/src/main/java/com/olympus/apollo/services/DeletionService.java index ca0c2a5..beaf3c4 100644 --- a/src/main/java/com/olympus/apollo/services/DeletionService.java +++ b/src/main/java/com/olympus/apollo/services/DeletionService.java @@ -1,6 +1,8 @@ package com.olympus.apollo.services; import com.olympus.apollo.dto.DeleteGitRepoDetailsRequest; +import com.olympus.apollo.dto.ResultDTO; +import com.olympus.apollo.exception.vectorStoreMetaDetailsEmptyException; import com.olympus.apollo.models.KSGitInfo; import com.olympus.apollo.models.KSGitIngestionInfo; import com.olympus.apollo.models.KSIngestionInfo; @@ -12,11 +14,13 @@ import com.olympus.apollo.dto.DeletionRequest; import com.olympus.apollo.dto.VectorStoreMetadataDetails; import com.olympus.apollo.models.VectorStore; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.messaging.simp.SimpMessagingTemplate; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import java.util.List; import java.util.Optional; +import java.util.concurrent.CompletableFuture; @Service public class DeletionService { @@ -38,6 +42,9 @@ public class DeletionService { @Autowired private VectorStoreRepository vectorStoreRepository; + @Autowired + private SimpMessagingTemplate simpMessagingTemplate; + @Async("asyncTaskExecutor") public void deleteRecords(DeletionRequest deletionRequest) { try { @@ -79,80 +86,143 @@ public class DeletionService { throw new RuntimeException("An error occurred while deleting records", e); } } - @Async("asyncTaskExecutor") - public void deleteRecordsOfGitRepo(DeleteGitRepoDetailsRequest deleteGitRepoDetailsRequest) { - try { - boolean KSGitInfoExists = deleteGitRepoDetailsRequest.getKsGitInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitInfoId().isEmpty() && ksGitInfoRepository.existsById(deleteGitRepoDetailsRequest.getKsGitInfoId()); - logger.info("KSGitInfo with id {} exists.", deleteGitRepoDetailsRequest.getKsGitInfoId()); - boolean KSGitIngestionInfoExists = deleteGitRepoDetailsRequest.getKsGitIngestionInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitIngestionInfoId().isEmpty() && ksGitIngestionInfoRepository.existsById(deleteGitRepoDetailsRequest.getKsGitIngestionInfoId()); - logger.info("KSGitIngestionInfo with id {} exists.", deleteGitRepoDetailsRequest.getKsGitIngestionInfoId()); - boolean vectorStoreGitDetailsExists = deleteGitRepoDetailsRequest.getKsApplicationName() != null && deleteGitRepoDetailsRequest.getKsDocSource() != null && deleteGitRepoDetailsRequest.getKsFileSource() != null && deleteGitRepoDetailsRequest.getKsDoctype() != null && deleteGitRepoDetailsRequest.getKsBranch() != null; + @Async + public CompletableFuture deleteRecordsOfGitRepo(DeleteGitRepoDetailsRequest deleteGitRepoDetailsRequest) { + return CompletableFuture.runAsync(() -> { + ResultDTO resultDTO = new ResultDTO(); + try { + String ksGitInfoId = deleteGitRepoDetailsRequest.getKsGitInfoId(); + String ksGitIngestionInfoId = deleteGitRepoDetailsRequest.getKsGitIngestionInfoId(); + String applicationName=deleteGitRepoDetailsRequest.getKsApplicationName(); + String ksDocSource=deleteGitRepoDetailsRequest.getKsDocSource(); + String ksFileSource=deleteGitRepoDetailsRequest.getKsFileSource(); + String ksDoctype=deleteGitRepoDetailsRequest.getKsDoctype(); + String ksBranch=deleteGitRepoDetailsRequest.getKsBranch(); - Optional ksGitInfo = ksGitInfoRepository.findById(deleteGitRepoDetailsRequest.getKsGitInfoId()); - String ingestionStatus = ksGitInfo.get().getIngestionStatus(); - logger.info("Ingestion Status is {}.",ingestionStatus+" "+Thread.currentThread().getName()); - List vectorStoreMetadataDetails = vectorStoreGitDetailsExists ? vectorStoreRepository.findGitVectorByMetadata(deleteGitRepoDetailsRequest.getKsDoctype(), deleteGitRepoDetailsRequest.getKsDocSource(), deleteGitRepoDetailsRequest.getKsFileSource(), deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsBranch()) : List.of(); + boolean KSGitInfoExists = ksGitInfoId != null && !ksGitInfoId.isEmpty() && ksGitInfoRepository.existsById(ksGitInfoId); + boolean KSGitIngestionInfoExists = ksGitIngestionInfoId != null && !ksGitIngestionInfoId.isEmpty() && ksGitIngestionInfoRepository.existsById(ksGitIngestionInfoId); + boolean vectorStoreGitDetailsExists = applicationName != null && ksDocSource != null && ksFileSource != null && ksDoctype != null && ksBranch != null; - if (KSGitInfoExists && KSGitIngestionInfoExists) { - if(ingestionStatus.equals("ERROR")){ - if (deleteGitRepoDetailsRequest.getKsGitInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitInfoId().isEmpty()) { - ksGitInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitInfoId()); - logger.info("KsGitInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitInfoId()); - } + logger.info("KSGitInfo with id {} exists: {}", ksGitInfoId,KSGitInfoExists); + logger.info("KSGitIngestionInfo with id {} exists: {}", ksGitIngestionInfoId,KSGitIngestionInfoExists); - if (deleteGitRepoDetailsRequest.getKsGitIngestionInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitIngestionInfoId().isEmpty()) { - ksGitIngestionInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitIngestionInfoId()); - logger.info("KSGitIngestionInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitIngestionInfoId()); - } - if( !vectorStoreMetadataDetails.isEmpty()){ - for (VectorStore store : vectorStoreMetadataDetails) { - vectorStoreRepository.deleteById(store.getId()); - logger.info("VectorStore with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), store.getId()); - } - } + Optional ksGitInfoOpt = ksGitInfoRepository.findById(ksGitInfoId); + if(ksGitInfoOpt.isEmpty()){ + String message = applicationName + " With Branch " + ksBranch + " record deletion failed due to KSGitInfo with id "+ksGitInfoId+" does not exist."; + logger.warn(message); - } else if (ingestionStatus.equals("INGESTED") && !vectorStoreMetadataDetails.isEmpty()) { - if (deleteGitRepoDetailsRequest.getKsGitInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitInfoId().isEmpty()) { - ksGitInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitInfoId()); - logger.info("KsGitInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitInfoId()); - } - - if (deleteGitRepoDetailsRequest.getKsGitIngestionInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitIngestionInfoId().isEmpty()) { - ksGitIngestionInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitIngestionInfoId()); - logger.info("KSGitIngestionInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitIngestionInfoId()); - } - - for (VectorStore store : vectorStoreMetadataDetails) { - vectorStoreRepository.deleteById(store.getId()); - logger.info("VectorStore with id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), store.getId()); - } - logger.info("All records deleted successfully."); - }else if (ingestionStatus.equals("NEW") && vectorStoreMetadataDetails.isEmpty()) { - if (deleteGitRepoDetailsRequest.getKsGitInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitInfoId().isEmpty()) { - ksGitInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitInfoId()); - logger.info("KsGitInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitInfoId()); - } - - if (deleteGitRepoDetailsRequest.getKsGitIngestionInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitIngestionInfoId().isEmpty()) { - ksGitIngestionInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitIngestionInfoId()); - logger.info("KSGitIngestionInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitIngestionInfoId()); - } - logger.info("records deleted successfully."); - } - - } else { - if (!KSGitInfoExists) { - logger.warn("getKsGitInfo with reponame {} and id {} does not exist.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitInfoId()); - } else if (!KSGitIngestionInfoExists) { - logger.warn("KSGitIngestionInfo with reponame {} and id {} does not exist.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitIngestionInfoId()); - } else if (vectorStoreMetadataDetails.isEmpty()) { - logger.warn("No VectorStore Data available"); + resultDTO.setSuccess(false); + resultDTO.setMessage(message); + simpMessagingTemplate.convertAndSend("/topic/deletion-status",resultDTO); + return; } + + KSGitInfo ksGitInfo = ksGitInfoOpt.get(); + String ingestionStatus = ksGitInfo.getIngestionStatus(); + logger.info("Ingestion Status is {}.", ingestionStatus); + + List vectorStoreMetadataDetails = vectorStoreGitDetailsExists + ? vectorStoreRepository.findGitVectorByMetadata(ksDoctype,ksDocSource, ksFileSource, applicationName, ksBranch) + : List.of(); + + if (KSGitInfoExists && KSGitIngestionInfoExists) { + deleteRecordsBasedOnIngestionStatus(ksGitInfoId,ksBranch,ingestionStatus,ksGitIngestionInfoId,vectorStoreMetadataDetails,applicationName); + + String message = applicationName + " With Branch " + ksBranch + " records removed successfully having KSGitInfo with id "+ksGitInfoId; + logger.info(message); + + resultDTO.setSuccess(true); + resultDTO.setMessage(applicationName + " With Branch " + ksBranch + " records removed successfully "); + simpMessagingTemplate.convertAndSend("/topic/deletion-status",resultDTO); + } else { + if (!KSGitInfoExists) { + String message = applicationName + " With Branch " + ksBranch + " record deletion failed due to KSGitInfo with id "+ksGitInfoId+" does not exist."; + logger.error(message); + + resultDTO.setSuccess(false); + resultDTO.setMessage(message); + simpMessagingTemplate.convertAndSend("/topic/deletion-status",resultDTO); + } else if (!KSGitIngestionInfoExists) { + String message = applicationName + " With Branch " + ksBranch + " record deletion failed due to KSGitIngestionInfo with id "+ksGitIngestionInfoId+" does not exist."; + logger.error(message); + + resultDTO.setSuccess(false); + resultDTO.setMessage(message); + simpMessagingTemplate.convertAndSend("/topic/deletion-status",resultDTO); + } else if (vectorStoreMetadataDetails.isEmpty()) { + String message = applicationName + " With Branch " + ksBranch + " record deletion failed due to No VectorStore Data available"; + logger.error(message); + + resultDTO.setSuccess(false); + resultDTO.setMessage(message); + simpMessagingTemplate.convertAndSend("/topic/deletion-status",resultDTO); + } + } + } catch (Exception e) { + String message = "An error occurred while deleting records"; + logger.error(message,e); + + resultDTO.setSuccess(false); + resultDTO.setMessage(message); + simpMessagingTemplate.convertAndSend("/topic/deletion-status",resultDTO); + + throw new RuntimeException("An error occurred while deleting records", e); + } + }); + } + + private void deleteRecordsBasedOnIngestionStatus(String ksGitInfoId,String ksBranch,String ingestionStatus,String ksGitIngestionInfoId,List vectorStoreMetaDetails,String applicationName){ + try{ + switch (ingestionStatus){ + case "INGESTION-ERROR": + case "INGESTION-IN-PROGRESS": + case "INGESTED": + deleteGitInfoAndIngestionInfo(ksGitInfoId,ksGitIngestionInfoId,applicationName); + deleteVectorStores(vectorStoreMetaDetails,applicationName); + break; + case "REPO-NEW": + case "REPO-CLONE-IN-PROGRESS": + case "REPO-CLONE-COMPLETED": + case "REPO-CLONE-FAILED": + if (vectorStoreMetaDetails.isEmpty()) { + deleteGitInfoAndIngestionInfo(ksGitInfoId, ksGitIngestionInfoId, applicationName); + }else { + // Throw a custom exception if vectorStoreMetaDetails is not empty + throw new vectorStoreMetaDetailsEmptyException("VectorStoreMetaDetails is not empty for application name "+applicationName+" branch "+ksBranch+" and ingestion status is " + ingestionStatus); + } + break; + default: + logger.warn("Unknown ingestion status: {}", ingestionStatus); + } + } catch (vectorStoreMetaDetailsEmptyException e){ + logger.error("vectorStoreMetaDetailsEmptyException occurred: ", e); + throw e; + } catch (Exception e){ + logger.error("An error occurred while deleting records based on ingestion status: ", e); + throw new RuntimeException("An error occurred while deleting records based on ingestion status", e); + } + } + + private void deleteGitInfoAndIngestionInfo(String ksGitInfoId,String ksGitIngestionInfoID,String applicationName){ + if (ksGitInfoId != null && !ksGitInfoId.isEmpty()) { + ksGitInfoRepository.deleteById(ksGitInfoId); + logger.info("KsGitInfo with reponame {} and id {} deleted successfully.", applicationName, ksGitInfoId); + } + + if (ksGitIngestionInfoID != null && !ksGitIngestionInfoID.isEmpty()) { + ksGitIngestionInfoRepository.deleteById(ksGitIngestionInfoID); + logger.info("KSGitIngestionInfo with reponame {} and id {} deleted successfully.", applicationName, ksGitIngestionInfoID); + } + } + + private void deleteVectorStores(List vectorStoreMetadataDetails, String applicationName){ + if(!vectorStoreMetadataDetails.isEmpty()){ + for (VectorStore store : vectorStoreMetadataDetails) { + String storeId=store.getId(); + vectorStoreRepository.deleteById(storeId); + logger.info("VectorStore with id {} deleted successfully.", applicationName, storeId); } - } catch (Exception e) { - logger.error("An error occurred while deleting records: ", e); - throw new RuntimeException("An error occurred while deleting records", e); } } } \ No newline at end of file diff --git a/src/main/java/com/olympus/apollo/services/FileSystemStorageService.java b/src/main/java/com/olympus/apollo/services/FileSystemStorageService.java index 17c776c..65065b2 100644 --- a/src/main/java/com/olympus/apollo/services/FileSystemStorageService.java +++ b/src/main/java/com/olympus/apollo/services/FileSystemStorageService.java @@ -7,13 +7,11 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; import java.util.stream.Stream; +import com.olympus.apollo.exception.StorageException; +import com.olympus.apollo.exception.StorageFileNotFoundException; +import com.olympus.apollo.properties.StorageProperties; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.Resource; import org.springframework.core.io.UrlResource; @@ -30,7 +28,7 @@ public class FileSystemStorageService implements StorageService { public FileSystemStorageService(StorageProperties properties) { if(properties.getLocation().trim().length() == 0){ - throw new StorageException("File upload location can not be Empty."); + throw new StorageException("File upload location can not be Empty."); } this.rootLocation = Paths.get(properties.getLocation()); diff --git a/src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java b/src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java index 40b05e6..43a39a2 100644 --- a/src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java +++ b/src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java @@ -13,6 +13,10 @@ import java.util.concurrent.CompletableFuture; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.olympus.apollo.dto.GitCloneOutput; +import com.olympus.apollo.dto.ResultDTO; +import com.olympus.apollo.exception.BranchCheckoutException; +import com.olympus.apollo.exception.GitCloneException; import com.olympus.apollo.repository.VectorStoreRepository; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.errors.GitAPIException; @@ -26,6 +30,8 @@ import org.springframework.ai.transformer.splitter.TokenTextSplitter; import org.springframework.ai.vectorstore.VectorStore; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.ResponseEntity; +import org.springframework.messaging.simp.SimpMessagingTemplate; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; @@ -51,6 +57,9 @@ public class GitRepositoryIngestor { @Autowired private GitService gitService; + @Autowired + private SimpMessagingTemplate simpMessagingTemplate; + public GitRepositoryIngestor(VectorStore vectorStore) { this.vectorStore = vectorStore; } @@ -58,84 +67,137 @@ public class GitRepositoryIngestor { Logger logger = LoggerFactory.getLogger(GitRepositoryIngestor.class); @Async - public CompletableFuture ingestGitRepository(String repo,String branchName) { - Optional optionalDocument = ksGitInfoRepository.findByRepoNameAndBranchName(repo,branchName); - if (optionalDocument.isPresent()) { - KSGitInfo ksGitInfo = optionalDocument.get(); - if ("NEW".equals(ksGitInfo.getIngestionStatus()) || "ERROR".equals(ksGitInfo.getIngestionStatus())) { - ingestRepo(repo,branchName, ksGitInfo); - } else { - logger.info("OOPS: Document is already Injected"); + public CompletableFuture ingestGitRepository(String repo, String branchName) { + return CompletableFuture.runAsync(() -> { + try { + Optional optionalDocument = ksGitInfoRepository.findByRepoNameAndBranchName(repo,branchName); + if (optionalDocument.isPresent()) { + KSGitInfo ksGitInfo = optionalDocument.get(); + if ("REPO-CLONE-COMPLETED".equals(ksGitInfo.getIngestionStatus())) { + ResultDTO repoResult = ingestRepo(repo,branchName, ksGitInfo); + // Notify clients via WebSocket or other mechanisms + //notifyClients(repo, branchName, repoResult); + } else { + String message = "Document is already ingested or has a different status: " + ksGitInfo.getIngestionStatus(); + logger.info(message); + } + } else { + String message = "Document not found for repo: " + repo + " branch: " + branchName; + logger.info(message); + } + } catch (Exception e){ + String errorMessage = "UnExpected Error: "+ e.getMessage(); + logger.error(errorMessage,e); + // Handle exception and notify clients + //notifyClients(repo, branchName, new ResultDTO(false, errorMessage)); } - } else { - logger.info("OOPS: Document Not found"); - } - return CompletableFuture.completedFuture(null); + }); } - private void ingestRepo(String repoName,String branchName, KSGitInfo ksGitInfo) { - String repoPath = basePath+"/"+ repoName + "/"; - //String repoPath = basePath + "\\" + repoName + "\\"; //need to modify before deploy - gitService.checkOutRepository(repoName,branchName); - logger.info("Repository path : " + repoPath); - try (Git git = Git.open(new File(repoPath))) { - ksGitInfo.setIngestionStatus("IN PROGRESS"); + private ResultDTO ingestRepo(String repoName,String branchName, KSGitInfo ksGitInfo) { - KSGitIngestionInfo ingestionInfo = ksGitInfo.getKsGitIngestionInfo(); - logger.info("Metadata : " + ingestionInfo.getMetadata()); - ksGitInfoRepository.save(ksGitInfo); + String repoPath = basePath + File.separator + repoName; + logger.info("Starting ingestion for repo: {} and branch: {}", repoName, branchName); + ResultDTO resultDTO = new ResultDTO(); - Repository repository = git.getRepository(); - RevCommit latestCommit = git.log().setMaxCount(1).call().iterator().next(); + try { + gitService.checkOutRepository(repoName, branchName); + logger.info("Repository path of Ingestion : {}", repoPath); - try (TreeWalk treeWalk = new TreeWalk(repository)) { - treeWalk.addTree(latestCommit.getTree()); - treeWalk.setRecursive(true); + try (Git git = Git.open(new File(repoPath))) { + ksGitInfo.setIngestionStatus("INGESTION-IN-PROGRESS"); + ksGitInfoRepository.save(ksGitInfo); - List documents = new ArrayList<>(); + KSGitIngestionInfo ingestionInfo = ksGitInfo.getKsGitIngestionInfo(); + logger.info("Metadata of ingestionInfo: {}",ingestionInfo.getMetadata()); - while (treeWalk.next()) { - String filePath = treeWalk.getPathString(); - String fileName = treeWalk.getNameString(); + Repository repository = git.getRepository(); + RevCommit latestCommit = git.log().setMaxCount(1).call().iterator().next(); - if (isRelevantFile(fileName)) { - byte[] fileContent = repository.open(treeWalk.getObjectId(0)).getBytes(); - String fileContentStr = new String(fileContent, StandardCharsets.UTF_8); + try (TreeWalk treeWalk = new TreeWalk(repository)) { + treeWalk.addTree(latestCommit.getTree()); + treeWalk.setRecursive(true); - Map metadata = extractMetadata(fileName, fileContentStr); - metadata.put("filePath", filePath); - metadata.put("fileName", fileName); + List documents = new ArrayList<>(); - Document doc = new Document(fileContentStr); - doc.getMetadata().putAll(metadata); + while (treeWalk.next()) { + String filePath = treeWalk.getPathString(); + String fileName = treeWalk.getNameString(); - doc.getMetadata().putAll(ingestionInfo.getMetadata()); - documents.add(doc); + if (isRelevantFile(fileName)) { + byte[] fileContent = repository.open(treeWalk.getObjectId(0)).getBytes(); + String fileContentStr = new String(fileContent, StandardCharsets.UTF_8); + + Map metadata = extractMetadata(fileName, fileContentStr); + metadata.put("filePath", filePath); + metadata.put("fileName", fileName); + + Document doc = new Document(fileContentStr); + doc.getMetadata().putAll(metadata); + doc.getMetadata().putAll(ingestionInfo.getMetadata()); + documents.add(doc); + } } + + TokenTextSplitter splitter = new TokenTextSplitter( + ingestionInfo.getDefaultChunkSize(), + ingestionInfo.getMinChunkSize(), + ingestionInfo.getMinChunkSizeToEmbed(), + ingestionInfo.getMaxNumberOfChunks(), + false + ); + + List splitDocuments = splitter.split(documents); + logger.info("Number of documents to be embedded: {}", splitDocuments.size()); + vectorStore.add(splitDocuments); + logger.info("Documents embedded Successfully"); + + } catch (IOException e) { + ksGitInfo.setIngestionStatus("INGESTION-ERROR"); + ksGitInfo.setIngestionDate(new Date()); + ksGitInfoRepository.save(ksGitInfo); + + logger.error("Error processing repository content", e); + resultDTO.setSuccess(false); + resultDTO.setMessage("Error processing repository content: " + e.getMessage()); + return resultDTO; } - TokenTextSplitter splitter = new TokenTextSplitter(ingestionInfo.getDefaultChunkSize(), - ingestionInfo.getMinChunkSize(), ingestionInfo.getMinChunkSizeToEmbed(), - ingestionInfo.getMaxNumberOfChunks(), false); + ksGitInfo.setIngestionStatus("INGESTED"); + ksGitInfo.setIngestionDate(new Date()); + ksGitInfoRepository.save(ksGitInfo); - List splitDocuments = splitter.split(documents); - logger.info("Number of documents: " + splitDocuments.size()); - vectorStore.add(splitDocuments); - logger.info("Documents embedded"); + resultDTO.setSuccess(true); + resultDTO.setMessage("Ingestion completed successfully"); + + } catch (IOException e) { + ksGitInfo.setIngestionStatus("INGESTION-ERROR"); + ksGitInfoRepository.save(ksGitInfo); + + logger.error("Error opening repository", e); + resultDTO.setSuccess(false); + resultDTO.setMessage("Error opening repository: " + e.getMessage()); } + }catch (BranchCheckoutException e){ + ksGitInfo.setIngestionStatus("INGESTION-ERROR"); + ksGitInfoRepository.save(ksGitInfo); - ksGitInfo.setIngestionStatus("INGESTED"); - ksGitInfo.setIngestionDate(new Date()); - ksGitInfoRepository.save(ksGitInfo); + logger.error("Error checking out repository branch", e); + resultDTO.setSuccess(false); + resultDTO.setMessage("Error checking out repository branch: " + e.getMessage()); } catch (Exception e) { - ksGitInfo.setIngestionStatus("ERROR"); + ksGitInfo.setIngestionStatus("INGESTION-ERROR"); ksGitInfoRepository.save(ksGitInfo); + logger.error("Error during ingestion", e); + resultDTO.setSuccess(false); + resultDTO.setMessage("Unexpected error during ingestion: " + e.getMessage()); } + return resultDTO; } - public CompletableFuture ReIngestGitRepository(String repo,String branchName) throws GitAPIException, IOException { + public CompletableFuture ReIngestGitRepository(String repo,String branchName) throws GitAPIException, IOException, BranchCheckoutException { Optional optionalDocument = ksGitInfoRepository.findByRepoNameAndBranchName(repo,branchName); if (optionalDocument.isPresent()) { KSGitInfo ksGitInfo = optionalDocument.get(); @@ -150,7 +212,7 @@ public class GitRepositoryIngestor { return CompletableFuture.completedFuture(null); } - private void reIngestRepo(String repoName,String branchName, KSGitInfo ksGitInfo) throws IOException, GitAPIException { + private void reIngestRepo(String repoName,String branchName, KSGitInfo ksGitInfo) throws IOException, GitAPIException, BranchCheckoutException { HashMap modifiedFiles = ksGitInfo.getGitModifiedFiles(); @@ -192,8 +254,7 @@ public class GitRepositoryIngestor { } } gitService.checkOutRepository(repoName,branchName); - String repoPath = basePath+"/"+ repoName + "/"; - //String repoPath = basePath+ "\\" + repoName + "\\"; //need to modify before deploy + String repoPath = basePath+ File.separator + repoName; logger.info("Repository path : " + repoPath); try (Git git = Git.open(new File(repoPath))) { diff --git a/src/main/java/com/olympus/apollo/services/GitService.java b/src/main/java/com/olympus/apollo/services/GitService.java index eaf997f..1f1666b 100644 --- a/src/main/java/com/olympus/apollo/services/GitService.java +++ b/src/main/java/com/olympus/apollo/services/GitService.java @@ -2,33 +2,40 @@ package com.olympus.apollo.services; import com.olympus.apollo.dto.GitCloneOutput; import com.olympus.apollo.dto.GitPullOutput; +import com.olympus.apollo.dto.ResultDTO; +import com.olympus.apollo.exception.BranchCheckoutException; +import com.olympus.apollo.exception.GitCloneException; import com.olympus.apollo.models.KSGitInfo; import com.olympus.apollo.repository.KSGitInfoRepository; -import com.olympus.apollo.repository.VectorStoreRepository; +import com.olympus.apollo.utils.GitUrlUtils; +import com.olympus.apollo.utils.GitUtils; + import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.RefNotFoundException; import org.eclipse.jgit.diff.DiffEntry; import org.eclipse.jgit.diff.DiffFormatter; import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.eclipse.jgit.transport.FetchResult; import org.eclipse.jgit.treewalk.AbstractTreeIterator; import org.eclipse.jgit.treewalk.CanonicalTreeParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.messaging.simp.SimpMessagingTemplate; +import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import java.io.File; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.CompletableFuture; @Service public class GitService { @@ -47,113 +54,156 @@ public class GitService { private String gitHubToken="23"; - public GitCloneOutput cloneRepository(String Source,String repoName,String branchName,String group, String tokenType) throws GitAPIException { - GitCloneOutput gitCloneOutput = new GitCloneOutput(); - try { - String gitlabToken; - String remoteRepoUrl; - switch (tokenType) { - case "cloud": - gitlabToken = cloudGitlabToken; - remoteRepoUrl = getProtocol(Source) + "gitlab-ci-token:" + gitlabToken + "@" + getDomain(Source) + "/" + group + "/" + repoName + ".git"; - break; - case "onpremises": - gitlabToken = onPremisesGitlabToken; - remoteRepoUrl = getProtocol(Source) + "gitlab-ci-token:" + gitlabToken + "@" + getDomain(Source) + "/" + group + "/" + repoName + ".git"; - break; - case "github": - gitlabToken = gitHubToken; - remoteRepoUrl = Source + "/" + group + "/" + repoName + ".git"; - break; - default: - gitCloneOutput.setMessage("tokenType is invalid"); - gitCloneOutput.setRepoName(repoName); - return gitCloneOutput; - } - System.out.println("remoteUrl : " + remoteRepoUrl); - //System.out.println("gitlab token : " + gitlabToken); - String localPath = basePath; - System.out.println("localpath"+basePath); - File cloneDirectory = new File(localPath,repoName); - //Files.createDirectories(Paths.get(localPath)); - //File cloneDirectory = new File(localPath, repoName); - //System.out.println("cloneDirectory : " + cloneDirectory); - if(!cloneDirectory.exists()){ - if (tokenType.equals("github")) { - Git.cloneRepository().setURI(remoteRepoUrl).setDirectory(cloneDirectory).call(); - } else { - Git.cloneRepository().setURI(remoteRepoUrl).setDirectory(cloneDirectory).setCredentialsProvider(new UsernamePasswordCredentialsProvider("username", gitlabToken)).call(); - } - }else { - System.out.println("Directory already exists. Skipping clone."); - } + @Autowired + private SimpMessagingTemplate simpMessagingTemplate; + + private static final Logger logger = LoggerFactory.getLogger(GitService.class); - checkOutRepository(repoName, branchName); - - gitCloneOutput.setRepoName(repoName); - gitCloneOutput.setMessage(repoName + "With Branch " + branchName + " cloned successfully"); - - }catch (Exception e){ - gitCloneOutput.setRepoName(repoName); - gitCloneOutput.setMessage("Error occured : "+e.getMessage()); - } - return gitCloneOutput; + private String buildGitURL(String source,String token,String group,String repoName){ + return GitUrlUtils.getProtocol(source) + "gitlab-ci-token:" + token + "@" + GitUrlUtils.getDomain(source) + "/" + group + "/" + repoName + ".git"; } - public String checkOutRepository(String repoName, String branchName) { - String localPath = basePath; - File repoDirectory = new File(localPath,repoName); + private void clone(String tokenType,String remoteRepoUrl,File cloneDirectory,String token) throws GitAPIException{ + if ("github".equals(tokenType)) { + Git.cloneRepository() + .setURI(remoteRepoUrl) + .setDirectory(cloneDirectory) + .call(); + } else { + Git.cloneRepository() + .setURI(remoteRepoUrl) + .setDirectory(cloneDirectory) + .setCredentialsProvider(new UsernamePasswordCredentialsProvider("username", token)) + .call(); + } + } + + public String checkOutRepository(String repoName, String branchName) throws BranchCheckoutException { + File repoDirectory = new File(basePath,repoName); try(Git git = Git.open(repoDirectory)) { Repository repository = git.getRepository(); - if(!branchExists(branchName,repository)) { + if(!GitUtils.branchExists(branchName,repository)) { FetchResult fetchResult = git.fetch().call(); - System.out.println("Fetch result: " + fetchResult.getMessages()); - git.checkout().setCreateBranch(true).setName(branchName).setStartPoint("origin/"+branchName).call(); + logger.info("Fetch result: {}",fetchResult.getMessages()); + git.checkout() + .setCreateBranch(true) + .setName(branchName) + .setStartPoint("origin/"+branchName) + .call(); + logger.info("Checked out new branch: {}", branchName); }else { - git.checkout().setName(branchName).setCreateBranch(false).call(); - System.out.println("Checked out existing branch: " + branchName); + git.checkout() + .setName(branchName) + .setCreateBranch(false) + .call(); + logger.info("Checked out existing branch: {}", branchName); } return branchName+" checkout successful"; } catch (RefNotFoundException e) { - try(Git git = Git.open(repoDirectory)) { - git.checkout().setName("origin/"+branchName).setCreateBranch(false).call(); - return branchName+" checkout successful"; - } catch (RefNotFoundException ex) { - return "checkout failed with error "+e.getMessage(); - }catch (Exception ex2){ - System.out.println("An error occurred: "+ ex2.getMessage()); - return "checkout failed with error "+ ex2.getMessage(); - } + String errorMessage = "Branch Not Found: " + e.getMessage(); + logger.error(errorMessage, e); + throw new BranchCheckoutException(errorMessage, e); } catch (Exception e){ - System.out.println("An error occurred: "+ e.getMessage()); - return "checkout failed with error "+e.getMessage(); + String errorMessage = "Checkout Failed with an error: " + e.getMessage(); + logger.error(errorMessage, e); + throw new BranchCheckoutException("Checkout failed with error: " + e.getMessage(), e); } } - private static boolean branchExists(String branchName,Repository repository){ - try{ - List refs = repository.getRefDatabase().getRefs(); + @Async + public CompletableFuture cloneRepository(String Source, String repoName, String branchName, String group, String tokenType,KSGitInfo ksGitInfo) { + return CompletableFuture.runAsync(() -> { + ksGitInfo.setIngestionStatus("REPO-CLONE-IN-PROGRESS"); + ksGitInfoRepository.save(ksGitInfo); - for(Ref ref : refs){ - System.out.println("refs "+ref.getName()); - } + ResultDTO resultDTO = new ResultDTO(); + try { + String gitlabToken; + String remoteRepoUrl; + switch (tokenType) { + case "cloud": + gitlabToken = cloudGitlabToken; + remoteRepoUrl = buildGitURL(Source,gitlabToken,group,repoName); + break; + case "onpremises": + gitlabToken = onPremisesGitlabToken; + remoteRepoUrl = buildGitURL(Source,gitlabToken,group,repoName); + break; + case "github": + gitlabToken = gitHubToken; + remoteRepoUrl = Source + "/" + group + "/" + repoName + ".git"; + break; + default: + String errorMessage = repoName + "With Branch " + branchName + " having invalid tokenType. "; + logger.error(errorMessage); - for(Ref ref : refs){ - if(ref.getName().equals("refs/heads/"+branchName)){ - return true; + throw new GitCloneException(errorMessage); } + logger.info("Remote URL: {}", remoteRepoUrl); + + File cloneDirectory = new File(basePath,repoName); + if(!cloneDirectory.exists()){ + clone(tokenType,remoteRepoUrl,cloneDirectory,gitlabToken); + }else { + logger.info("Directory already exists. Skipping clone."); + } + + + String checkoutMessage = checkOutRepository(repoName, branchName); + + ksGitInfo.setIngestionStatus("REPO-CLONE-COMPLETED"); + ksGitInfoRepository.save(ksGitInfo); + + resultDTO.setSuccess(true); + resultDTO.setMessage(repoName + " With Branch " + branchName + " cloned successfully. "+checkoutMessage); + simpMessagingTemplate.convertAndSend("/topic/clone-status",resultDTO); + }catch (GitAPIException e){ + String errorMessage = "Git API error: " + e.getMessage(); + logger.error(errorMessage, e); + + ksGitInfo.setIngestionStatus("REPO-CLONE-FAILED"); + ksGitInfoRepository.save(ksGitInfo); + + resultDTO.setSuccess(false); + resultDTO.setMessage(repoName + " With Branch " + branchName + " clone Failed. "+errorMessage); + simpMessagingTemplate.convertAndSend("/topic/clone-status",resultDTO); + } catch (BranchCheckoutException e){ + String errorMessage = "Branch Checkout Error: "+ e.getMessage(); + logger.error(errorMessage,e); + + ksGitInfo.setIngestionStatus("REPO-CLONE-FAILED"); + ksGitInfoRepository.save(ksGitInfo); + + resultDTO.setSuccess(false); + resultDTO.setMessage(repoName + " With Branch " + branchName + " clone Failed. "+errorMessage); + simpMessagingTemplate.convertAndSend("/topic/clone-status",resultDTO); + } catch (GitCloneException e){ + String errorMessage = "Git Clone Error: "+ e.getMessage(); + logger.error(errorMessage,e); + + ksGitInfo.setIngestionStatus("REPO-CLONE-FAILED"); + ksGitInfoRepository.save(ksGitInfo); + + resultDTO.setSuccess(false); + resultDTO.setMessage(repoName + " With Branch " + branchName + " clone Failed. "+errorMessage); + simpMessagingTemplate.convertAndSend("/topic/clone-status",resultDTO); + } catch (Exception e){ + String errorMessage = "UnExpected Error: "+ e.getMessage(); + logger.error(errorMessage,e); + + ksGitInfo.setIngestionStatus("REPO-CLONE-FAILED"); + ksGitInfoRepository.save(ksGitInfo); + + resultDTO.setSuccess(false); + resultDTO.setMessage(repoName + " With Branch " + branchName + " clone Failed. "+errorMessage); + simpMessagingTemplate.convertAndSend("/topic/clone-status",resultDTO); } - return false; - } catch (IOException e) { - throw new RuntimeException(e); - } + }); } public GitPullOutput pullChanges(String repoName,String branchName) { - String localPath= basePath; - - File repoDirectory = new File(localPath,repoName); + File repoDirectory = new File(basePath,repoName); GitPullOutput gitPullOutput =new GitPullOutput(); Map gitdiff=null; try(Git git = Git.open(repoDirectory)) { @@ -236,15 +286,5 @@ public class GitService { } } - // Method to extract the protocol part - private static String getProtocol(String url) { - int protocolEndIndex = url.indexOf("://"); - return protocolEndIndex != -1 ? url.substring(0, protocolEndIndex + 3) : ""; - } - // Method to extract the domain part - private static String getDomain(String url) { - int protocolEndIndex = url.indexOf("://"); - return protocolEndIndex != -1 ? url.substring(protocolEndIndex + 3) : url; - } } diff --git a/src/main/java/com/olympus/apollo/utils/GitUrlUtils.java b/src/main/java/com/olympus/apollo/utils/GitUrlUtils.java new file mode 100644 index 0000000..6cd3422 --- /dev/null +++ b/src/main/java/com/olympus/apollo/utils/GitUrlUtils.java @@ -0,0 +1,32 @@ +package com.olympus.apollo.utils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.MalformedURLException; +import java.net.URL; + +public class GitUrlUtils { + public static final Logger logger = LoggerFactory.getLogger(GitUrlUtils.class); + // Method to extract the protocol part + public static String getProtocol(String url) { + try{ + URL urlObject = new URL(url); + return urlObject.getProtocol() +"://"; + } catch (MalformedURLException e){ + logger.error("Invalid URL for protocol extraction: {}",url,e); + return ""; + } + } + + // Method to extract the domain part + public static String getDomain(String url) { + try{ + URL urlObject = new URL(url); + return urlObject.getHost(); + } catch (MalformedURLException e) { + logger.error("Invalid URL for domain extraction: {}", url, e); + return url; // Returning the original URL if the domain extraction fails + } + } +} diff --git a/src/main/java/com/olympus/apollo/utils/GitUtils.java b/src/main/java/com/olympus/apollo/utils/GitUtils.java new file mode 100644 index 0000000..24806c3 --- /dev/null +++ b/src/main/java/com/olympus/apollo/utils/GitUtils.java @@ -0,0 +1,85 @@ +package com.olympus.apollo.utils; + +import com.olympus.apollo.dto.GitCloneInput; +import com.olympus.apollo.models.KSGitInfo; +import com.olympus.apollo.models.KSGitIngestionInfo; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.Repository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.List; + +public class GitUtils { + + private static final Logger logger = LoggerFactory.getLogger(GitUtils.class); + + public static KSGitInfo createKSGitInfo(GitCloneInput gitCloneInput,String basePath){ + logger.debug("Creating KSGitInfo with repoName: {}, branch: {}",gitCloneInput.getRepoName(),gitCloneInput.getBranch()); + + KSGitInfo info = new KSGitInfo(); + info.setRepoName(gitCloneInput.getRepoName()); + info.setBranch(gitCloneInput.getBranch()); + info.setCommitId(gitCloneInput.getCommitId()); + info.setRepoPath(basePath+ File.separator +gitCloneInput.getBranch()); + info.setIngestionStatus("REPO-NEW"); + info.setIngestionDate(new Date()); + info.setIngestionDateFormat(new SimpleDateFormat("MM/dd/yy").format(new Date())); + + logger.info("KSGitInfo Created: {}, {}",info.getRepoName(),info.getBranch()); + return info; + } + + public static KSGitIngestionInfo createKSGitIngestionInfo(GitCloneInput gitCloneInput){ + logger.debug("Creating KSGitIngestionInfo with repoName: {}, branch: {}",gitCloneInput.getRepoName(),gitCloneInput.getBranch()); + + HashMap metaData = createMetaData(gitCloneInput.getRepoName(), gitCloneInput.getBranch()); + KSGitIngestionInfo ksGitIngestionInfo = new KSGitIngestionInfo(); + ksGitIngestionInfo.setMetadata(metaData); + ksGitIngestionInfo.setMinChunkSizeToEmbed(gitCloneInput.getMinChunkSizeToEmbed()); + ksGitIngestionInfo.setMaxNumberOfChunks(gitCloneInput.getMaxNumberOfChunks()); + ksGitIngestionInfo.setMinChunkSize(gitCloneInput.getMinChunkSize()); + ksGitIngestionInfo.setDefaultChunkSize(gitCloneInput.getDefaultChunkSize()); + + logger.info("ksGitIngestionInfo Created: {}, {}",gitCloneInput.getRepoName(),gitCloneInput.getBranch()); + return ksGitIngestionInfo; + } + + private static HashMap createMetaData(String repoName, String branchName){ + logger.debug("Creating metadata for repoName: {}, branchName: {}", repoName, branchName); + + HashMap metaData =new HashMap<>(); + metaData.put("KsApplicationName", repoName); + metaData.put("KsDoctype", "gitrepository"); + metaData.put("KsDocSource", "gitlab"); + metaData.put("KsFileSource", repoName); + + metaData.put("KsBranch", branchName); + metaData.put("KsRepoName", repoName); + + logger.info("Metadata created: {}", metaData); + return metaData; + } + + public static boolean branchExists(String branchName, Repository repository){ + try{ + List refs = repository.getRefDatabase().getRefs(); + + for(Ref ref : refs){ + if(ref.getName().equals("refs/heads/"+branchName)){ + return true; + } + } + return false; + } catch (IOException e) { + logger.error("Error checking if branch exists: {}", e.getMessage(), e); + throw new RuntimeException("Error accessing the repository", e); + } + } + +}