admin/apollo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

clone repo & delete git records websocket done GitService&DeletionService

Browse Source
This commit is contained in:
sumedh
2024-09-12 23:08:12 +05:30
parent 1679ab8dba
commit 8ef069eca8
21 changed files with 700 additions and 328 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.gitignore vendored
View File

@@ -33,8 +33,6 @@ build/
.vscode/ .vscode/
##changes in filepath before deploy ##changes in filepath before deploy
src/main/java/com/olympus/apollo/services/StorageProperties.java #src/main/java/com/olympus/apollo/services/StorageProperties.java
src/main/resources/application.properties #src/main/resources/application.properties
src/main/java/com/olympus/apollo/services/GitService.java
src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java

5
pom.xml
View File

@@ -37,6 +37,11 @@
<artifactId>spring-boot-starter-web</artifactId> <artifactId>spring-boot-starter-web</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-websocket</artifactId>
</dependency>
<dependency> <dependency>
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-mongodb</artifactId> <artifactId>spring-boot-starter-data-mongodb</artifactId>

2
src/main/java/com/olympus/apollo/ApolloApplication.java
View File

@@ -5,7 +5,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableAsync;
import com.olympus.apollo.services.StorageProperties; import com.olympus.apollo.properties.StorageProperties;
@SpringBootApplication @SpringBootApplication
@EnableConfigurationProperties(StorageProperties.class) @EnableConfigurationProperties(StorageProperties.class)

25
src/main/java/com/olympus/apollo/config/WebSocketConfig.java Normal file
View File

@@ -0,0 +1,25 @@
package com.olympus.apollo.config;
import org.springframework.context.annotation.Configuration;
import org.springframework.messaging.simp.config.MessageBrokerRegistry;
import org.springframework.web.socket.config.annotation.EnableWebSocketMessageBroker;
import org.springframework.web.socket.config.annotation.StompEndpointRegistry;
import org.springframework.web.socket.config.annotation.WebSocketMessageBrokerConfigurer;
@Configuration
@EnableWebSocketMessageBroker
public class WebSocketConfig implements WebSocketMessageBrokerConfigurer {
@Override
public void registerStompEndpoints(StompEndpointRegistry registry) {
registry.addEndpoint("/ws/endpoint")
.setAllowedOrigins("http://apollo.olympusai.live")
.withSockJS();
}
@Override
public void configureMessageBroker(MessageBrokerRegistry config) {
config.enableSimpleBroker("/topic"); //should be used as prefix while sending request from spring to browser
config.setApplicationDestinationPrefixes("/app"); //should be used as prefix from browser to spring server
}
}

152
src/main/java/com/olympus/apollo/controllers/FeApi/KSGitController.java
View File

@@ -5,12 +5,19 @@ import java.text.SimpleDateFormat;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.concurrent.CompletableFuture;
import com.olympus.apollo.dto.*; import com.olympus.apollo.dto.*;
import com.olympus.apollo.exception.GitCloneException;
import com.olympus.apollo.services.GitService; import com.olympus.apollo.services.GitService;
import com.olympus.apollo.utils.GitUtils;
import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.GitAPIException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
import org.springframework.http.HttpStatusCode;
import org.springframework.http.ResponseEntity; import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*; import org.springframework.web.bind.annotation.*;
@@ -24,94 +31,79 @@ import com.olympus.apollo.services.KSGitInfoService;
@RequestMapping("/fe-api/ks_git_repos") @RequestMapping("/fe-api/ks_git_repos")
public class KSGitController { public class KSGitController {
@Autowired @Autowired
private KSGitInfoRepository ksGitInfoRepository; private KSGitInfoRepository ksGitInfoRepository;
@Autowired @Autowired
private KSGitIngestionInfoRepository ksGitIngestionInfoRepository; private KSGitIngestionInfoRepository ksGitIngestionInfoRepository;
@Autowired @Autowired
private GitService gitService; private GitService gitService;
@Value("${gitlab.path}") @Value("${gitlab.path}")
private String basePath; private String basePath;
@GetMapping("") private static final Logger logger = LoggerFactory.getLogger(KSGitController.class);
public List<KSGitInfo> listGitInfo() {
List<KSGitInfo> result = (List<KSGitInfo>) ksGitInfoRepository.findAll();
return result;
}
@PostMapping("/uploadRepo") @GetMapping("")
public ResponseEntity<String> handleGitUpload(@RequestBody KSGitUploadDTO ksGitUploadDTO) { public List<KSGitInfo> listGitInfo() {
List<KSGitInfo> result = (List<KSGitInfo>) ksGitInfoRepository.findAll();
return result;
}
KSGitInfo ksGitInfo = new KSGitInfo(); @PostMapping("/uploadRepo")
ksGitInfo.setRepoName(ksGitUploadDTO.getRepoName()); public ResponseEntity<String> handleGitUpload(@RequestBody KSGitUploadDTO ksGitUploadDTO) {
ksGitInfo.setBranch(ksGitUploadDTO.getBranch());
ksGitInfo.setCommitId(ksGitUploadDTO.getCommitId());
ksGitInfo.setRepoPath(ksGitUploadDTO.getRepoPath());
ksGitInfo.setIngestionStatus("NEW");
ksGitInfo.setIngestionDate(new Date());
ksGitInfo.setIngestionDateFormat(new SimpleDateFormat("MM/dd/yy").format(new Date()));
KSGitIngestionInfo ksGitIngestionInfo = new KSGitIngestionInfo(); KSGitInfo ksGitInfo = new KSGitInfo();
HashMap<String, String> metadata = new HashMap<>(); ksGitInfo.setRepoName(ksGitUploadDTO.getRepoName());
ksGitInfo.setBranch(ksGitUploadDTO.getBranch());
metadata.put("KsApplicationName", ksGitUploadDTO.getRepoName()); ksGitInfo.setCommitId(ksGitUploadDTO.getCommitId());
metadata.put("KsDoctype", "gitrepository"); ksGitInfo.setRepoPath(ksGitUploadDTO.getRepoPath());
metadata.put("KsDocSource", "gitlab"); ksGitInfo.setIngestionStatus("NEW");
metadata.put("KsFileSource", ksGitUploadDTO.getRepoName()); ksGitInfo.setIngestionDate(new Date());
ksGitInfo.setIngestionDateFormat(new SimpleDateFormat("MM/dd/yy").format(new Date()));
metadata.put("KsBranch", ksGitUploadDTO.getBranch());
metadata.put("KsRepoName", ksGitUploadDTO.getRepoName());
ksGitIngestionInfo.setMetadata(metadata); KSGitIngestionInfo ksGitIngestionInfo = new KSGitIngestionInfo();
ksGitIngestionInfo.setMinChunkSizeToEmbed(ksGitUploadDTO.getMinChunkSizeToEmbed()); HashMap<String, String> metadata = new HashMap<>();
ksGitIngestionInfo.setMaxNumberOfChunks(ksGitUploadDTO.getMaxNumberOfChunks());
ksGitIngestionInfo.setMinChunkSize(ksGitUploadDTO.getMinChunkSize());
ksGitIngestionInfo.setDefaultChunkSize(ksGitUploadDTO.getDefaultChunkSize());
metadata.put("KsApplicationName", ksGitUploadDTO.getRepoName());
metadata.put("KsDoctype", "gitrepository");
metadata.put("KsDocSource", "gitlab");
metadata.put("KsFileSource", ksGitUploadDTO.getRepoName());
metadata.put("KsBranch", ksGitUploadDTO.getBranch());
metadata.put("KsRepoName", ksGitUploadDTO.getRepoName());
ksGitIngestionInfo.setMetadata(metadata);
ksGitIngestionInfo.setMinChunkSizeToEmbed(ksGitUploadDTO.getMinChunkSizeToEmbed());
ksGitIngestionInfo.setMaxNumberOfChunks(ksGitUploadDTO.getMaxNumberOfChunks());
ksGitIngestionInfo.setMinChunkSize(ksGitUploadDTO.getMinChunkSize());
ksGitIngestionInfo.setDefaultChunkSize(ksGitUploadDTO.getDefaultChunkSize());
ksGitIngestionInfoRepository.save(ksGitIngestionInfo);
ksGitInfo.setKsGitIngestionInfo(ksGitIngestionInfo);
ksGitInfoRepository.save(ksGitInfo);
return ResponseEntity.ok("Upload successful");
}
//clone the repository
@PostMapping("/clone")
public ResponseEntity<ResultDTO> gitClone(@RequestBody GitCloneInput gitCloneInput) throws GitCloneException {
KSGitInfo ksGitInfo = GitUtils.createKSGitInfo(gitCloneInput, basePath);
KSGitIngestionInfo ksGitIngestionInfo = GitUtils.createKSGitIngestionInfo(gitCloneInput);
ksGitIngestionInfoRepository.save(ksGitIngestionInfo); ksGitIngestionInfoRepository.save(ksGitIngestionInfo);
ksGitInfo.setKsGitIngestionInfo(ksGitIngestionInfo); ksGitInfo.setKsGitIngestionInfo(ksGitIngestionInfo);
ksGitInfoRepository.save(ksGitInfo); ksGitInfoRepository.save(ksGitInfo);
return ResponseEntity.ok("Upload successful"); // Start the async processing
} gitService.cloneRepository(gitCloneInput.getSource(), gitCloneInput.getRepoName(), gitCloneInput.getBranch(), gitCloneInput.getGroup(), gitCloneInput.getTokenType(), ksGitInfo);
//clone the master branch from remote repository //Return an immediate response
@PostMapping("/clone") ResultDTO response = new ResultDTO();
public GitCloneOutput gitClone(@RequestBody GitCloneInput gitCloneInput) throws GitAPIException, IOException { response.setMessage("Ingestion process initiated for repo: " + gitCloneInput.getRepoName() + " branch: " + gitCloneInput.getBranch());
KSGitInfo ksGitInfo = new KSGitInfo(); response.setSuccess(true);
ksGitInfo.setRepoName(gitCloneInput.getRepoName()); return ResponseEntity.ok(response);
ksGitInfo.setBranch(gitCloneInput.getBranch()); }
ksGitInfo.setCommitId(gitCloneInput.getCommitId());
ksGitInfo.setRepoPath(basePath+"/"+gitCloneInput.getBranch());
ksGitInfo.setIngestionStatus("NEW");
ksGitInfo.setIngestionDate(new Date());
ksGitInfo.setIngestionDateFormat(new SimpleDateFormat("MM/dd/yy").format(new Date()));
KSGitIngestionInfo ksGitIngestionInfo = new KSGitIngestionInfo();
HashMap<String, String> metadata = new HashMap<>();
metadata.put("KsApplicationName", gitCloneInput.getRepoName());
metadata.put("KsDoctype", "gitrepository");
metadata.put("KsDocSource", "gitlab");
metadata.put("KsFileSource", gitCloneInput.getRepoName());
metadata.put("KsBranch", gitCloneInput.getBranch());
metadata.put("KsRepoName", gitCloneInput.getRepoName());
ksGitIngestionInfo.setMetadata(metadata);
ksGitIngestionInfo.setMinChunkSizeToEmbed(gitCloneInput.getMinChunkSizeToEmbed());
ksGitIngestionInfo.setMaxNumberOfChunks(gitCloneInput.getMaxNumberOfChunks());
ksGitIngestionInfo.setMinChunkSize(gitCloneInput.getMinChunkSize());
ksGitIngestionInfo.setDefaultChunkSize(gitCloneInput.getDefaultChunkSize());
ksGitIngestionInfoRepository.save(ksGitIngestionInfo);
ksGitInfo.setKsGitIngestionInfo(ksGitIngestionInfo);
ksGitInfoRepository.save(ksGitInfo);
return gitService.cloneRepository(gitCloneInput.getSource(),gitCloneInput.getRepoName(),gitCloneInput.getBranch(),gitCloneInput.getGroup(),gitCloneInput.getTokenType());
}
/* /*
curl --location 'http://localhost:8082/fe-api/ks_git_repos/clone' \ curl --location 'http://localhost:8082/fe-api/ks_git_repos/clone' \
--header 'Content-Type: application/json' \ --header 'Content-Type: application/json' \
@@ -120,10 +112,10 @@ public class KSGitController {
' '
*/ */
//pull latest changes from master branch //pull latest changes from master branch
@GetMapping("/pullchanges") @GetMapping("/pullchanges")
public GitPullOutput gitPull(@RequestParam String repoName,@RequestParam String branchName){ public GitPullOutput gitPull(@RequestParam String repoName, @RequestParam String branchName) {
return gitService.pullChanges(repoName,branchName); return gitService.pullChanges(repoName, branchName);
} }
} }

2
src/main/java/com/olympus/apollo/controllers/KSFileController.java
View File

@@ -13,7 +13,7 @@ import com.olympus.apollo.models.KSDocument;
import com.olympus.apollo.models.KSIngestionInfo; import com.olympus.apollo.models.KSIngestionInfo;
import com.olympus.apollo.repository.KSDocumentRepository; import com.olympus.apollo.repository.KSDocumentRepository;
import com.olympus.apollo.repository.KSIngestionInfoRepository; import com.olympus.apollo.repository.KSIngestionInfoRepository;
import com.olympus.apollo.services.StorageFileNotFoundException; import com.olympus.apollo.exception.StorageFileNotFoundException;
import com.olympus.apollo.services.StorageService; import com.olympus.apollo.services.StorageService;
import com.olympus.apollo.dto.FileUploadDTO; import com.olympus.apollo.dto.FileUploadDTO;

20
src/main/java/com/olympus/apollo/controllers/TestController.java
View File

@@ -1,7 +1,9 @@
package com.olympus.apollo.controllers; package com.olympus.apollo.controllers;
import java.util.List; import java.util.List;
import java.util.concurrent.CompletableFuture;
import com.olympus.apollo.dto.ResultDTO;
import com.olympus.apollo.services.GitService; import com.olympus.apollo.services.GitService;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@@ -55,17 +57,15 @@ public class TestController {
} }
@GetMapping("test/ingest_repo") @GetMapping("test/ingest_repo")
public ResponseEntity<String> ingestRepo(@RequestParam String repoName,@RequestParam String branchName) { public ResponseEntity<ResultDTO> ingestRepo(@RequestParam String repoName, @RequestParam String branchName) {
try { // Start the async processing
gitRepositoryIngestor.ingestGitRepository(repoName,branchName); gitRepositoryIngestor.ingestGitRepository(repoName,branchName);
return ResponseEntity.ok("Ingestion Started");
} catch (Exception e) {
logger.error("Error during ingestion start", e); //Return an immediate response
ResultDTO response = new ResultDTO();
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) response.setMessage("Ingestion process initiated for repo: " + repoName + " branch: " + branchName);
.body("Error starting ingestion: " + e.getMessage()); response.setSuccess(true);
} return ResponseEntity.ok(response);
} }
@GetMapping("test/reingest_repo") @GetMapping("test/reingest_repo")

10
src/main/java/com/olympus/apollo/dto/ResultDTO.java Normal file
View File

@@ -0,0 +1,10 @@
package com.olympus.apollo.dto;
import lombok.Getter;
import lombok.Setter;
@Setter @Getter
public class ResultDTO {
private boolean success;
private String message;
}

11
src/main/java/com/olympus/apollo/exception/BranchCheckoutException.java Normal file
View File

@@ -0,0 +1,11 @@
package com.olympus.apollo.exception;
public class BranchCheckoutException extends Exception{
public BranchCheckoutException(String message){
super(message);
}
public BranchCheckoutException(String message,Throwable cause){
super(message,cause);
}
}

11
src/main/java/com/olympus/apollo/exception/GitCloneException.java Normal file
View File

@@ -0,0 +1,11 @@
package com.olympus.apollo.exception;
public class GitCloneException extends Exception{
public GitCloneException(String message){
super(message);
}
public GitCloneException(String message,Throwable cause){
super(message,cause);
}
}

25
src/main/java/com/olympus/apollo/exception/GlobalExceptionHandler.java Normal file
View File

@@ -0,0 +1,25 @@
package com.olympus.apollo.exception;
import com.olympus.apollo.dto.GitCloneOutput;
import org.slf4j.LoggerFactory;
import org.slf4j.Logger;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.ControllerAdvice;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.ResponseBody;
@ControllerAdvice
public class GlobalExceptionHandler {
private static final Logger logger = LoggerFactory.getLogger(GlobalExceptionHandler.class);
@ExceptionHandler(GitCloneException.class)
@ResponseBody
public ResponseEntity<GitCloneOutput> handleGitCloneException(GitCloneException ex) {
logger.error("Git clone exception: {}", ex.getMessage(), ex);
GitCloneOutput output = new GitCloneOutput();
output.setMessage("Git clone error: " + ex.getMessage());
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(output);
}
}

2
src/main/java/com/olympus/apollo/services/StorageException.java → src/main/java/com/olympus/apollo/exception/StorageException.java
View File

@@ -1,4 +1,4 @@
package com.olympus.apollo.services; package com.olympus.apollo.exception;
public class StorageException extends RuntimeException { public class StorageException extends RuntimeException {
public StorageException(String message) { public StorageException(String message) {

2
src/main/java/com/olympus/apollo/services/StorageFileNotFoundException.java → src/main/java/com/olympus/apollo/exception/StorageFileNotFoundException.java
View File

@@ -1,4 +1,4 @@
package com.olympus.apollo.services; package com.olympus.apollo.exception;
public class StorageFileNotFoundException extends StorageException { public class StorageFileNotFoundException extends StorageException {

10
src/main/java/com/olympus/apollo/exception/vectorStoreMetaDetailsEmptyException.java Normal file
View File

@@ -0,0 +1,10 @@
package com.olympus.apollo.exception;
public class vectorStoreMetaDetailsEmptyException extends RuntimeException{
public vectorStoreMetaDetailsEmptyException(String message){
super(message);
}
public vectorStoreMetaDetailsEmptyException(String message,Throwable cause){
super(message,cause);
}
}

3
src/main/java/com/olympus/apollo/services/StorageProperties.java → src/main/java/com/olympus/apollo/properties/StorageProperties.java
View File

@@ -1,7 +1,6 @@
package com.olympus.apollo.services; package com.olympus.apollo.properties;
import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.beans.factory.annotation.Value;
@ConfigurationProperties("storage") @ConfigurationProperties("storage")
public class StorageProperties { public class StorageProperties {

204
src/main/java/com/olympus/apollo/services/DeletionService.java
View File

@@ -1,6 +1,8 @@
package com.olympus.apollo.services; package com.olympus.apollo.services;
import com.olympus.apollo.dto.DeleteGitRepoDetailsRequest; import com.olympus.apollo.dto.DeleteGitRepoDetailsRequest;
import com.olympus.apollo.dto.ResultDTO;
import com.olympus.apollo.exception.vectorStoreMetaDetailsEmptyException;
import com.olympus.apollo.models.KSGitInfo; import com.olympus.apollo.models.KSGitInfo;
import com.olympus.apollo.models.KSGitIngestionInfo; import com.olympus.apollo.models.KSGitIngestionInfo;
import com.olympus.apollo.models.KSIngestionInfo; import com.olympus.apollo.models.KSIngestionInfo;
@@ -12,11 +14,13 @@ import com.olympus.apollo.dto.DeletionRequest;
import com.olympus.apollo.dto.VectorStoreMetadataDetails; import com.olympus.apollo.dto.VectorStoreMetadataDetails;
import com.olympus.apollo.models.VectorStore; import com.olympus.apollo.models.VectorStore;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.messaging.simp.SimpMessagingTemplate;
import org.springframework.scheduling.annotation.Async; import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
import java.util.concurrent.CompletableFuture;
@Service @Service
public class DeletionService { public class DeletionService {
@@ -38,6 +42,9 @@ public class DeletionService {
@Autowired @Autowired
private VectorStoreRepository vectorStoreRepository; private VectorStoreRepository vectorStoreRepository;
@Autowired
private SimpMessagingTemplate simpMessagingTemplate;
@Async("asyncTaskExecutor") @Async("asyncTaskExecutor")
public void deleteRecords(DeletionRequest deletionRequest) { public void deleteRecords(DeletionRequest deletionRequest) {
try { try {
@@ -79,80 +86,143 @@ public class DeletionService {
throw new RuntimeException("An error occurred while deleting records", e); throw new RuntimeException("An error occurred while deleting records", e);
} }
} }
@Async("asyncTaskExecutor")
public void deleteRecordsOfGitRepo(DeleteGitRepoDetailsRequest deleteGitRepoDetailsRequest) {
try {
boolean KSGitInfoExists = deleteGitRepoDetailsRequest.getKsGitInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitInfoId().isEmpty() && ksGitInfoRepository.existsById(deleteGitRepoDetailsRequest.getKsGitInfoId());
logger.info("KSGitInfo with id {} exists.", deleteGitRepoDetailsRequest.getKsGitInfoId());
boolean KSGitIngestionInfoExists = deleteGitRepoDetailsRequest.getKsGitIngestionInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitIngestionInfoId().isEmpty() && ksGitIngestionInfoRepository.existsById(deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
logger.info("KSGitIngestionInfo with id {} exists.", deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
boolean vectorStoreGitDetailsExists = deleteGitRepoDetailsRequest.getKsApplicationName() != null && deleteGitRepoDetailsRequest.getKsDocSource() != null && deleteGitRepoDetailsRequest.getKsFileSource() != null && deleteGitRepoDetailsRequest.getKsDoctype() != null && deleteGitRepoDetailsRequest.getKsBranch() != null;
@Async
public CompletableFuture<Void> deleteRecordsOfGitRepo(DeleteGitRepoDetailsRequest deleteGitRepoDetailsRequest) {
return CompletableFuture.runAsync(() -> {
ResultDTO resultDTO = new ResultDTO();
try {
String ksGitInfoId = deleteGitRepoDetailsRequest.getKsGitInfoId();
String ksGitIngestionInfoId = deleteGitRepoDetailsRequest.getKsGitIngestionInfoId();
String applicationName=deleteGitRepoDetailsRequest.getKsApplicationName();
String ksDocSource=deleteGitRepoDetailsRequest.getKsDocSource();
String ksFileSource=deleteGitRepoDetailsRequest.getKsFileSource();
String ksDoctype=deleteGitRepoDetailsRequest.getKsDoctype();
String ksBranch=deleteGitRepoDetailsRequest.getKsBranch();
Optional<KSGitInfo> ksGitInfo = ksGitInfoRepository.findById(deleteGitRepoDetailsRequest.getKsGitInfoId()); boolean KSGitInfoExists = ksGitInfoId != null && !ksGitInfoId.isEmpty() && ksGitInfoRepository.existsById(ksGitInfoId);
String ingestionStatus = ksGitInfo.get().getIngestionStatus(); boolean KSGitIngestionInfoExists = ksGitIngestionInfoId != null && !ksGitIngestionInfoId.isEmpty() && ksGitIngestionInfoRepository.existsById(ksGitIngestionInfoId);
logger.info("Ingestion Status is {}.",ingestionStatus+" "+Thread.currentThread().getName()); boolean vectorStoreGitDetailsExists = applicationName != null && ksDocSource != null && ksFileSource != null && ksDoctype != null && ksBranch != null;
List<VectorStore> vectorStoreMetadataDetails = vectorStoreGitDetailsExists ? vectorStoreRepository.findGitVectorByMetadata(deleteGitRepoDetailsRequest.getKsDoctype(), deleteGitRepoDetailsRequest.getKsDocSource(), deleteGitRepoDetailsRequest.getKsFileSource(), deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsBranch()) : List.of();
if (KSGitInfoExists && KSGitIngestionInfoExists) { logger.info("KSGitInfo with id {} exists: {}", ksGitInfoId,KSGitInfoExists);
if(ingestionStatus.equals("ERROR")){ logger.info("KSGitIngestionInfo with id {} exists: {}", ksGitIngestionInfoId,KSGitIngestionInfoExists);
if (deleteGitRepoDetailsRequest.getKsGitInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitInfoId().isEmpty()) {
ksGitInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitInfoId());
logger.info("KsGitInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitInfoId());
}
if (deleteGitRepoDetailsRequest.getKsGitIngestionInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitIngestionInfoId().isEmpty()) { Optional<KSGitInfo> ksGitInfoOpt = ksGitInfoRepository.findById(ksGitInfoId);
ksGitIngestionInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitIngestionInfoId()); if(ksGitInfoOpt.isEmpty()){
logger.info("KSGitIngestionInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitIngestionInfoId()); String message = applicationName + " With Branch " + ksBranch + " record deletion failed due to KSGitInfo with id "+ksGitInfoId+" does not exist.";
} logger.warn(message);
if( !vectorStoreMetadataDetails.isEmpty()){
for (VectorStore store : vectorStoreMetadataDetails) {
vectorStoreRepository.deleteById(store.getId());
logger.info("VectorStore with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), store.getId());
}
}
} else if (ingestionStatus.equals("INGESTED") && !vectorStoreMetadataDetails.isEmpty()) { resultDTO.setSuccess(false);
if (deleteGitRepoDetailsRequest.getKsGitInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitInfoId().isEmpty()) { resultDTO.setMessage(message);
ksGitInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitInfoId()); simpMessagingTemplate.convertAndSend("/topic/deletion-status",resultDTO);
logger.info("KsGitInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitInfoId()); return;
}
if (deleteGitRepoDetailsRequest.getKsGitIngestionInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitIngestionInfoId().isEmpty()) {
ksGitIngestionInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
logger.info("KSGitIngestionInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
}
for (VectorStore store : vectorStoreMetadataDetails) {
vectorStoreRepository.deleteById(store.getId());
logger.info("VectorStore with id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), store.getId());
}
logger.info("All records deleted successfully.");
}else if (ingestionStatus.equals("NEW") && vectorStoreMetadataDetails.isEmpty()) {
if (deleteGitRepoDetailsRequest.getKsGitInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitInfoId().isEmpty()) {
ksGitInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitInfoId());
logger.info("KsGitInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitInfoId());
}
if (deleteGitRepoDetailsRequest.getKsGitIngestionInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitIngestionInfoId().isEmpty()) {
ksGitIngestionInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
logger.info("KSGitIngestionInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
}
logger.info("records deleted successfully.");
}
} else {
if (!KSGitInfoExists) {
logger.warn("getKsGitInfo with reponame {} and id {} does not exist.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitInfoId());
} else if (!KSGitIngestionInfoExists) {
logger.warn("KSGitIngestionInfo with reponame {} and id {} does not exist.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
} else if (vectorStoreMetadataDetails.isEmpty()) {
logger.warn("No VectorStore Data available");
} }
KSGitInfo ksGitInfo = ksGitInfoOpt.get();
String ingestionStatus = ksGitInfo.getIngestionStatus();
logger.info("Ingestion Status is {}.", ingestionStatus);
List<VectorStore> vectorStoreMetadataDetails = vectorStoreGitDetailsExists
? vectorStoreRepository.findGitVectorByMetadata(ksDoctype,ksDocSource, ksFileSource, applicationName, ksBranch)
: List.of();
if (KSGitInfoExists && KSGitIngestionInfoExists) {
deleteRecordsBasedOnIngestionStatus(ksGitInfoId,ksBranch,ingestionStatus,ksGitIngestionInfoId,vectorStoreMetadataDetails,applicationName);
String message = applicationName + " With Branch " + ksBranch + " records removed successfully having KSGitInfo with id "+ksGitInfoId;
logger.info(message);
resultDTO.setSuccess(true);
resultDTO.setMessage(applicationName + " With Branch " + ksBranch + " records removed successfully ");
simpMessagingTemplate.convertAndSend("/topic/deletion-status",resultDTO);
} else {
if (!KSGitInfoExists) {
String message = applicationName + " With Branch " + ksBranch + " record deletion failed due to KSGitInfo with id "+ksGitInfoId+" does not exist.";
logger.error(message);
resultDTO.setSuccess(false);
resultDTO.setMessage(message);
simpMessagingTemplate.convertAndSend("/topic/deletion-status",resultDTO);
} else if (!KSGitIngestionInfoExists) {
String message = applicationName + " With Branch " + ksBranch + " record deletion failed due to KSGitIngestionInfo with id "+ksGitIngestionInfoId+" does not exist.";
logger.error(message);
resultDTO.setSuccess(false);
resultDTO.setMessage(message);
simpMessagingTemplate.convertAndSend("/topic/deletion-status",resultDTO);
} else if (vectorStoreMetadataDetails.isEmpty()) {
String message = applicationName + " With Branch " + ksBranch + " record deletion failed due to No VectorStore Data available";
logger.error(message);
resultDTO.setSuccess(false);
resultDTO.setMessage(message);
simpMessagingTemplate.convertAndSend("/topic/deletion-status",resultDTO);
}
}
} catch (Exception e) {
String message = "An error occurred while deleting records";
logger.error(message,e);
resultDTO.setSuccess(false);
resultDTO.setMessage(message);
simpMessagingTemplate.convertAndSend("/topic/deletion-status",resultDTO);
throw new RuntimeException("An error occurred while deleting records", e);
}
});
}
private void deleteRecordsBasedOnIngestionStatus(String ksGitInfoId,String ksBranch,String ingestionStatus,String ksGitIngestionInfoId,List<VectorStore> vectorStoreMetaDetails,String applicationName){
try{
switch (ingestionStatus){
case "INGESTION-ERROR":
case "INGESTION-IN-PROGRESS":
case "INGESTED":
deleteGitInfoAndIngestionInfo(ksGitInfoId,ksGitIngestionInfoId,applicationName);
deleteVectorStores(vectorStoreMetaDetails,applicationName);
break;
case "REPO-NEW":
case "REPO-CLONE-IN-PROGRESS":
case "REPO-CLONE-COMPLETED":
case "REPO-CLONE-FAILED":
if (vectorStoreMetaDetails.isEmpty()) {
deleteGitInfoAndIngestionInfo(ksGitInfoId, ksGitIngestionInfoId, applicationName);
}else {
// Throw a custom exception if vectorStoreMetaDetails is not empty
throw new vectorStoreMetaDetailsEmptyException("VectorStoreMetaDetails is not empty for application name "+applicationName+" branch "+ksBranch+" and ingestion status is " + ingestionStatus);
}
break;
default:
logger.warn("Unknown ingestion status: {}", ingestionStatus);
}
} catch (vectorStoreMetaDetailsEmptyException e){
logger.error("vectorStoreMetaDetailsEmptyException occurred: ", e);
throw e;
} catch (Exception e){
logger.error("An error occurred while deleting records based on ingestion status: ", e);
throw new RuntimeException("An error occurred while deleting records based on ingestion status", e);
}
}
private void deleteGitInfoAndIngestionInfo(String ksGitInfoId,String ksGitIngestionInfoID,String applicationName){
if (ksGitInfoId != null && !ksGitInfoId.isEmpty()) {
ksGitInfoRepository.deleteById(ksGitInfoId);
logger.info("KsGitInfo with reponame {} and id {} deleted successfully.", applicationName, ksGitInfoId);
}
if (ksGitIngestionInfoID != null && !ksGitIngestionInfoID.isEmpty()) {
ksGitIngestionInfoRepository.deleteById(ksGitIngestionInfoID);
logger.info("KSGitIngestionInfo with reponame {} and id {} deleted successfully.", applicationName, ksGitIngestionInfoID);
}
}
private void deleteVectorStores(List<VectorStore> vectorStoreMetadataDetails, String applicationName){
if(!vectorStoreMetadataDetails.isEmpty()){
for (VectorStore store : vectorStoreMetadataDetails) {
String storeId=store.getId();
vectorStoreRepository.deleteById(storeId);
logger.info("VectorStore with id {} deleted successfully.", applicationName, storeId);
} }
} catch (Exception e) {
logger.error("An error occurred while deleting records: ", e);
throw new RuntimeException("An error occurred while deleting records", e);
} }
} }
} }

10
src/main/java/com/olympus/apollo/services/FileSystemStorageService.java
View File

@@ -7,13 +7,11 @@ import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.nio.file.StandardCopyOption; import java.nio.file.StandardCopyOption;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import com.olympus.apollo.exception.StorageException;
import com.olympus.apollo.exception.StorageFileNotFoundException;
import com.olympus.apollo.properties.StorageProperties;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.Resource; import org.springframework.core.io.Resource;
import org.springframework.core.io.UrlResource; import org.springframework.core.io.UrlResource;
@@ -30,7 +28,7 @@ public class FileSystemStorageService implements StorageService {
public FileSystemStorageService(StorageProperties properties) { public FileSystemStorageService(StorageProperties properties) {
if(properties.getLocation().trim().length() == 0){ if(properties.getLocation().trim().length() == 0){
throw new StorageException("File upload location can not be Empty."); throw new StorageException("File upload location can not be Empty.");
} }
this.rootLocation = Paths.get(properties.getLocation()); this.rootLocation = Paths.get(properties.getLocation());

173
src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java
View File

@@ -13,6 +13,10 @@ import java.util.concurrent.CompletableFuture;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import com.olympus.apollo.dto.GitCloneOutput;
import com.olympus.apollo.dto.ResultDTO;
import com.olympus.apollo.exception.BranchCheckoutException;
import com.olympus.apollo.exception.GitCloneException;
import com.olympus.apollo.repository.VectorStoreRepository; import com.olympus.apollo.repository.VectorStoreRepository;
import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.GitAPIException;
@@ -26,6 +30,8 @@ import org.springframework.ai.transformer.splitter.TokenTextSplitter;
import org.springframework.ai.vectorstore.VectorStore; import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.ResponseEntity;
import org.springframework.messaging.simp.SimpMessagingTemplate;
import org.springframework.scheduling.annotation.Async; import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
@@ -51,6 +57,9 @@ public class GitRepositoryIngestor {
@Autowired @Autowired
private GitService gitService; private GitService gitService;
@Autowired
private SimpMessagingTemplate simpMessagingTemplate;
public GitRepositoryIngestor(VectorStore vectorStore) { public GitRepositoryIngestor(VectorStore vectorStore) {
this.vectorStore = vectorStore; this.vectorStore = vectorStore;
} }
@@ -58,84 +67,137 @@ public class GitRepositoryIngestor {
Logger logger = LoggerFactory.getLogger(GitRepositoryIngestor.class); Logger logger = LoggerFactory.getLogger(GitRepositoryIngestor.class);
@Async @Async
public CompletableFuture<Void> ingestGitRepository(String repo,String branchName) { public CompletableFuture<Void> ingestGitRepository(String repo, String branchName) {
Optional<KSGitInfo> optionalDocument = ksGitInfoRepository.findByRepoNameAndBranchName(repo,branchName); return CompletableFuture.runAsync(() -> {
if (optionalDocument.isPresent()) { try {
KSGitInfo ksGitInfo = optionalDocument.get(); Optional<KSGitInfo> optionalDocument = ksGitInfoRepository.findByRepoNameAndBranchName(repo,branchName);
if ("NEW".equals(ksGitInfo.getIngestionStatus()) || "ERROR".equals(ksGitInfo.getIngestionStatus())) { if (optionalDocument.isPresent()) {
ingestRepo(repo,branchName, ksGitInfo); KSGitInfo ksGitInfo = optionalDocument.get();
} else { if ("REPO-CLONE-COMPLETED".equals(ksGitInfo.getIngestionStatus())) {
logger.info("OOPS: Document is already Injected"); ResultDTO repoResult = ingestRepo(repo,branchName, ksGitInfo);
// Notify clients via WebSocket or other mechanisms
//notifyClients(repo, branchName, repoResult);
} else {
String message = "Document is already ingested or has a different status: " + ksGitInfo.getIngestionStatus();
logger.info(message);
}
} else {
String message = "Document not found for repo: " + repo + " branch: " + branchName;
logger.info(message);
}
} catch (Exception e){
String errorMessage = "UnExpected Error: "+ e.getMessage();
logger.error(errorMessage,e);
// Handle exception and notify clients
//notifyClients(repo, branchName, new ResultDTO(false, errorMessage));
} }
} else { });
logger.info("OOPS: Document Not found");
}
return CompletableFuture.completedFuture(null);
} }
private void ingestRepo(String repoName,String branchName, KSGitInfo ksGitInfo) { private ResultDTO ingestRepo(String repoName,String branchName, KSGitInfo ksGitInfo) {
String repoPath = basePath+"/"+ repoName + "/";
//String repoPath = basePath + "\\" + repoName + "\\"; //need to modify before deploy
gitService.checkOutRepository(repoName,branchName);
logger.info("Repository path : " + repoPath);
try (Git git = Git.open(new File(repoPath))) {
ksGitInfo.setIngestionStatus("IN PROGRESS");
KSGitIngestionInfo ingestionInfo = ksGitInfo.getKsGitIngestionInfo(); String repoPath = basePath + File.separator + repoName;
logger.info("Metadata : " + ingestionInfo.getMetadata()); logger.info("Starting ingestion for repo: {} and branch: {}", repoName, branchName);
ksGitInfoRepository.save(ksGitInfo); ResultDTO resultDTO = new ResultDTO();
Repository repository = git.getRepository(); try {
RevCommit latestCommit = git.log().setMaxCount(1).call().iterator().next(); gitService.checkOutRepository(repoName, branchName);
logger.info("Repository path of Ingestion : {}", repoPath);
try (TreeWalk treeWalk = new TreeWalk(repository)) { try (Git git = Git.open(new File(repoPath))) {
treeWalk.addTree(latestCommit.getTree()); ksGitInfo.setIngestionStatus("INGESTION-IN-PROGRESS");
treeWalk.setRecursive(true); ksGitInfoRepository.save(ksGitInfo);
List<Document> documents = new ArrayList<>(); KSGitIngestionInfo ingestionInfo = ksGitInfo.getKsGitIngestionInfo();
logger.info("Metadata of ingestionInfo: {}",ingestionInfo.getMetadata());
while (treeWalk.next()) { Repository repository = git.getRepository();
String filePath = treeWalk.getPathString(); RevCommit latestCommit = git.log().setMaxCount(1).call().iterator().next();
String fileName = treeWalk.getNameString();
if (isRelevantFile(fileName)) { try (TreeWalk treeWalk = new TreeWalk(repository)) {
byte[] fileContent = repository.open(treeWalk.getObjectId(0)).getBytes(); treeWalk.addTree(latestCommit.getTree());
String fileContentStr = new String(fileContent, StandardCharsets.UTF_8); treeWalk.setRecursive(true);
Map<String, String> metadata = extractMetadata(fileName, fileContentStr); List<Document> documents = new ArrayList<>();
metadata.put("filePath", filePath);
metadata.put("fileName", fileName);
Document doc = new Document(fileContentStr); while (treeWalk.next()) {
doc.getMetadata().putAll(metadata); String filePath = treeWalk.getPathString();
String fileName = treeWalk.getNameString();
doc.getMetadata().putAll(ingestionInfo.getMetadata()); if (isRelevantFile(fileName)) {
documents.add(doc); byte[] fileContent = repository.open(treeWalk.getObjectId(0)).getBytes();
String fileContentStr = new String(fileContent, StandardCharsets.UTF_8);
Map<String, String> metadata = extractMetadata(fileName, fileContentStr);
metadata.put("filePath", filePath);
metadata.put("fileName", fileName);
Document doc = new Document(fileContentStr);
doc.getMetadata().putAll(metadata);
doc.getMetadata().putAll(ingestionInfo.getMetadata());
documents.add(doc);
}
} }
TokenTextSplitter splitter = new TokenTextSplitter(
ingestionInfo.getDefaultChunkSize(),
ingestionInfo.getMinChunkSize(),
ingestionInfo.getMinChunkSizeToEmbed(),
ingestionInfo.getMaxNumberOfChunks(),
false
);
List<Document> splitDocuments = splitter.split(documents);
logger.info("Number of documents to be embedded: {}", splitDocuments.size());
vectorStore.add(splitDocuments);
logger.info("Documents embedded Successfully");
} catch (IOException e) {
ksGitInfo.setIngestionStatus("INGESTION-ERROR");
ksGitInfo.setIngestionDate(new Date());
ksGitInfoRepository.save(ksGitInfo);
logger.error("Error processing repository content", e);
resultDTO.setSuccess(false);
resultDTO.setMessage("Error processing repository content: " + e.getMessage());
return resultDTO;
} }
TokenTextSplitter splitter = new TokenTextSplitter(ingestionInfo.getDefaultChunkSize(), ksGitInfo.setIngestionStatus("INGESTED");
ingestionInfo.getMinChunkSize(), ingestionInfo.getMinChunkSizeToEmbed(), ksGitInfo.setIngestionDate(new Date());
ingestionInfo.getMaxNumberOfChunks(), false); ksGitInfoRepository.save(ksGitInfo);
List<Document> splitDocuments = splitter.split(documents); resultDTO.setSuccess(true);
logger.info("Number of documents: " + splitDocuments.size()); resultDTO.setMessage("Ingestion completed successfully");
vectorStore.add(splitDocuments);
logger.info("Documents embedded"); } catch (IOException e) {
ksGitInfo.setIngestionStatus("INGESTION-ERROR");
ksGitInfoRepository.save(ksGitInfo);
logger.error("Error opening repository", e);
resultDTO.setSuccess(false);
resultDTO.setMessage("Error opening repository: " + e.getMessage());
} }
}catch (BranchCheckoutException e){
ksGitInfo.setIngestionStatus("INGESTION-ERROR");
ksGitInfoRepository.save(ksGitInfo);
ksGitInfo.setIngestionStatus("INGESTED"); logger.error("Error checking out repository branch", e);
ksGitInfo.setIngestionDate(new Date()); resultDTO.setSuccess(false);
ksGitInfoRepository.save(ksGitInfo); resultDTO.setMessage("Error checking out repository branch: " + e.getMessage());
} catch (Exception e) { } catch (Exception e) {
ksGitInfo.setIngestionStatus("ERROR"); ksGitInfo.setIngestionStatus("INGESTION-ERROR");
ksGitInfoRepository.save(ksGitInfo); ksGitInfoRepository.save(ksGitInfo);
logger.error("Error during ingestion", e); logger.error("Error during ingestion", e);
resultDTO.setSuccess(false);
resultDTO.setMessage("Unexpected error during ingestion: " + e.getMessage());
} }
return resultDTO;
} }
public CompletableFuture<Void> ReIngestGitRepository(String repo,String branchName) throws GitAPIException, IOException { public CompletableFuture<Void> ReIngestGitRepository(String repo,String branchName) throws GitAPIException, IOException, BranchCheckoutException {
Optional<KSGitInfo> optionalDocument = ksGitInfoRepository.findByRepoNameAndBranchName(repo,branchName); Optional<KSGitInfo> optionalDocument = ksGitInfoRepository.findByRepoNameAndBranchName(repo,branchName);
if (optionalDocument.isPresent()) { if (optionalDocument.isPresent()) {
KSGitInfo ksGitInfo = optionalDocument.get(); KSGitInfo ksGitInfo = optionalDocument.get();
@@ -150,7 +212,7 @@ public class GitRepositoryIngestor {
return CompletableFuture.completedFuture(null); return CompletableFuture.completedFuture(null);
} }
private void reIngestRepo(String repoName,String branchName, KSGitInfo ksGitInfo) throws IOException, GitAPIException { private void reIngestRepo(String repoName,String branchName, KSGitInfo ksGitInfo) throws IOException, GitAPIException, BranchCheckoutException {
HashMap<String, String> modifiedFiles = ksGitInfo.getGitModifiedFiles(); HashMap<String, String> modifiedFiles = ksGitInfo.getGitModifiedFiles();
@@ -192,8 +254,7 @@ public class GitRepositoryIngestor {
} }
} }
gitService.checkOutRepository(repoName,branchName); gitService.checkOutRepository(repoName,branchName);
String repoPath = basePath+"/"+ repoName + "/"; String repoPath = basePath+ File.separator + repoName;
//String repoPath = basePath+ "\\" + repoName + "\\"; //need to modify before deploy
logger.info("Repository path : " + repoPath); logger.info("Repository path : " + repoPath);
try (Git git = Git.open(new File(repoPath))) { try (Git git = Git.open(new File(repoPath))) {

238
src/main/java/com/olympus/apollo/services/GitService.java
View File

@@ -2,33 +2,40 @@ package com.olympus.apollo.services;
import com.olympus.apollo.dto.GitCloneOutput; import com.olympus.apollo.dto.GitCloneOutput;
import com.olympus.apollo.dto.GitPullOutput; import com.olympus.apollo.dto.GitPullOutput;
import com.olympus.apollo.dto.ResultDTO;
import com.olympus.apollo.exception.BranchCheckoutException;
import com.olympus.apollo.exception.GitCloneException;
import com.olympus.apollo.models.KSGitInfo; import com.olympus.apollo.models.KSGitInfo;
import com.olympus.apollo.repository.KSGitInfoRepository; import com.olympus.apollo.repository.KSGitInfoRepository;
import com.olympus.apollo.repository.VectorStoreRepository; import com.olympus.apollo.utils.GitUrlUtils;
import com.olympus.apollo.utils.GitUtils;
import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.api.errors.RefNotFoundException; import org.eclipse.jgit.api.errors.RefNotFoundException;
import org.eclipse.jgit.diff.DiffEntry; import org.eclipse.jgit.diff.DiffEntry;
import org.eclipse.jgit.diff.DiffFormatter; import org.eclipse.jgit.diff.DiffFormatter;
import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.eclipse.jgit.transport.FetchResult; import org.eclipse.jgit.transport.FetchResult;
import org.eclipse.jgit.treewalk.AbstractTreeIterator; import org.eclipse.jgit.treewalk.AbstractTreeIterator;
import org.eclipse.jgit.treewalk.CanonicalTreeParser; import org.eclipse.jgit.treewalk.CanonicalTreeParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.messaging.simp.SimpMessagingTemplate;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.concurrent.CompletableFuture;
@Service @Service
public class GitService { public class GitService {
@@ -47,113 +54,156 @@ public class GitService {
private String gitHubToken="23"; private String gitHubToken="23";
public GitCloneOutput cloneRepository(String Source,String repoName,String branchName,String group, String tokenType) throws GitAPIException { @Autowired
GitCloneOutput gitCloneOutput = new GitCloneOutput(); private SimpMessagingTemplate simpMessagingTemplate;
try {
String gitlabToken; private static final Logger logger = LoggerFactory.getLogger(GitService.class);
String remoteRepoUrl;
switch (tokenType) {
case "cloud":
gitlabToken = cloudGitlabToken;
remoteRepoUrl = getProtocol(Source) + "gitlab-ci-token:" + gitlabToken + "@" + getDomain(Source) + "/" + group + "/" + repoName + ".git";
break;
case "onpremises":
gitlabToken = onPremisesGitlabToken;
remoteRepoUrl = getProtocol(Source) + "gitlab-ci-token:" + gitlabToken + "@" + getDomain(Source) + "/" + group + "/" + repoName + ".git";
break;
case "github":
gitlabToken = gitHubToken;
remoteRepoUrl = Source + "/" + group + "/" + repoName + ".git";
break;
default:
gitCloneOutput.setMessage("tokenType is invalid");
gitCloneOutput.setRepoName(repoName);
return gitCloneOutput;
}
System.out.println("remoteUrl : " + remoteRepoUrl);
//System.out.println("gitlab token : " + gitlabToken);
String localPath = basePath;
System.out.println("localpath"+basePath);
File cloneDirectory = new File(localPath,repoName);
//Files.createDirectories(Paths.get(localPath));
//File cloneDirectory = new File(localPath, repoName);
//System.out.println("cloneDirectory : " + cloneDirectory);
if(!cloneDirectory.exists()){
if (tokenType.equals("github")) {
Git.cloneRepository().setURI(remoteRepoUrl).setDirectory(cloneDirectory).call();
} else {
Git.cloneRepository().setURI(remoteRepoUrl).setDirectory(cloneDirectory).setCredentialsProvider(new UsernamePasswordCredentialsProvider("username", gitlabToken)).call();
}
}else {
System.out.println("Directory already exists. Skipping clone.");
}
checkOutRepository(repoName, branchName); private String buildGitURL(String source,String token,String group,String repoName){
return GitUrlUtils.getProtocol(source) + "gitlab-ci-token:" + token + "@" + GitUrlUtils.getDomain(source) + "/" + group + "/" + repoName + ".git";
gitCloneOutput.setRepoName(repoName);
gitCloneOutput.setMessage(repoName + "With Branch " + branchName + " cloned successfully");
}catch (Exception e){
gitCloneOutput.setRepoName(repoName);
gitCloneOutput.setMessage("Error occured : "+e.getMessage());
}
return gitCloneOutput;
} }
public String checkOutRepository(String repoName, String branchName) { private void clone(String tokenType,String remoteRepoUrl,File cloneDirectory,String token) throws GitAPIException{
String localPath = basePath; if ("github".equals(tokenType)) {
File repoDirectory = new File(localPath,repoName); Git.cloneRepository()
.setURI(remoteRepoUrl)
.setDirectory(cloneDirectory)
.call();
} else {
Git.cloneRepository()
.setURI(remoteRepoUrl)
.setDirectory(cloneDirectory)
.setCredentialsProvider(new UsernamePasswordCredentialsProvider("username", token))
.call();
}
}
public String checkOutRepository(String repoName, String branchName) throws BranchCheckoutException {
File repoDirectory = new File(basePath,repoName);
try(Git git = Git.open(repoDirectory)) { try(Git git = Git.open(repoDirectory)) {
Repository repository = git.getRepository(); Repository repository = git.getRepository();
if(!branchExists(branchName,repository)) { if(!GitUtils.branchExists(branchName,repository)) {
FetchResult fetchResult = git.fetch().call(); FetchResult fetchResult = git.fetch().call();
System.out.println("Fetch result: " + fetchResult.getMessages()); logger.info("Fetch result: {}",fetchResult.getMessages());
git.checkout().setCreateBranch(true).setName(branchName).setStartPoint("origin/"+branchName).call(); git.checkout()
.setCreateBranch(true)
.setName(branchName)
.setStartPoint("origin/"+branchName)
.call();
logger.info("Checked out new branch: {}", branchName);
}else { }else {
git.checkout().setName(branchName).setCreateBranch(false).call(); git.checkout()
System.out.println("Checked out existing branch: " + branchName); .setName(branchName)
.setCreateBranch(false)
.call();
logger.info("Checked out existing branch: {}", branchName);
} }
return branchName+" checkout successful"; return branchName+" checkout successful";
} catch (RefNotFoundException e) { } catch (RefNotFoundException e) {
try(Git git = Git.open(repoDirectory)) { String errorMessage = "Branch Not Found: " + e.getMessage();
git.checkout().setName("origin/"+branchName).setCreateBranch(false).call(); logger.error(errorMessage, e);
return branchName+" checkout successful"; throw new BranchCheckoutException(errorMessage, e);
} catch (RefNotFoundException ex) {
return "checkout failed with error "+e.getMessage();
}catch (Exception ex2){
System.out.println("An error occurred: "+ ex2.getMessage());
return "checkout failed with error "+ ex2.getMessage();
}
} catch (Exception e){ } catch (Exception e){
System.out.println("An error occurred: "+ e.getMessage()); String errorMessage = "Checkout Failed with an error: " + e.getMessage();
return "checkout failed with error "+e.getMessage(); logger.error(errorMessage, e);
throw new BranchCheckoutException("Checkout failed with error: " + e.getMessage(), e);
} }
} }
private static boolean branchExists(String branchName,Repository repository){ @Async
try{ public CompletableFuture<Void> cloneRepository(String Source, String repoName, String branchName, String group, String tokenType,KSGitInfo ksGitInfo) {
List<Ref> refs = repository.getRefDatabase().getRefs(); return CompletableFuture.runAsync(() -> {
ksGitInfo.setIngestionStatus("REPO-CLONE-IN-PROGRESS");
ksGitInfoRepository.save(ksGitInfo);
for(Ref ref : refs){ ResultDTO resultDTO = new ResultDTO();
System.out.println("refs "+ref.getName()); try {
} String gitlabToken;
String remoteRepoUrl;
switch (tokenType) {
case "cloud":
gitlabToken = cloudGitlabToken;
remoteRepoUrl = buildGitURL(Source,gitlabToken,group,repoName);
break;
case "onpremises":
gitlabToken = onPremisesGitlabToken;
remoteRepoUrl = buildGitURL(Source,gitlabToken,group,repoName);
break;
case "github":
gitlabToken = gitHubToken;
remoteRepoUrl = Source + "/" + group + "/" + repoName + ".git";
break;
default:
String errorMessage = repoName + "With Branch " + branchName + " having invalid tokenType. ";
logger.error(errorMessage);
for(Ref ref : refs){ throw new GitCloneException(errorMessage);
if(ref.getName().equals("refs/heads/"+branchName)){
return true;
} }
logger.info("Remote URL: {}", remoteRepoUrl);
File cloneDirectory = new File(basePath,repoName);
if(!cloneDirectory.exists()){
clone(tokenType,remoteRepoUrl,cloneDirectory,gitlabToken);
}else {
logger.info("Directory already exists. Skipping clone.");
}
String checkoutMessage = checkOutRepository(repoName, branchName);
ksGitInfo.setIngestionStatus("REPO-CLONE-COMPLETED");
ksGitInfoRepository.save(ksGitInfo);
resultDTO.setSuccess(true);
resultDTO.setMessage(repoName + " With Branch " + branchName + " cloned successfully. "+checkoutMessage);
simpMessagingTemplate.convertAndSend("/topic/clone-status",resultDTO);
}catch (GitAPIException e){
String errorMessage = "Git API error: " + e.getMessage();
logger.error(errorMessage, e);
ksGitInfo.setIngestionStatus("REPO-CLONE-FAILED");
ksGitInfoRepository.save(ksGitInfo);
resultDTO.setSuccess(false);
resultDTO.setMessage(repoName + " With Branch " + branchName + " clone Failed. "+errorMessage);
simpMessagingTemplate.convertAndSend("/topic/clone-status",resultDTO);
} catch (BranchCheckoutException e){
String errorMessage = "Branch Checkout Error: "+ e.getMessage();
logger.error(errorMessage,e);
ksGitInfo.setIngestionStatus("REPO-CLONE-FAILED");
ksGitInfoRepository.save(ksGitInfo);
resultDTO.setSuccess(false);
resultDTO.setMessage(repoName + " With Branch " + branchName + " clone Failed. "+errorMessage);
simpMessagingTemplate.convertAndSend("/topic/clone-status",resultDTO);
} catch (GitCloneException e){
String errorMessage = "Git Clone Error: "+ e.getMessage();
logger.error(errorMessage,e);
ksGitInfo.setIngestionStatus("REPO-CLONE-FAILED");
ksGitInfoRepository.save(ksGitInfo);
resultDTO.setSuccess(false);
resultDTO.setMessage(repoName + " With Branch " + branchName + " clone Failed. "+errorMessage);
simpMessagingTemplate.convertAndSend("/topic/clone-status",resultDTO);
} catch (Exception e){
String errorMessage = "UnExpected Error: "+ e.getMessage();
logger.error(errorMessage,e);
ksGitInfo.setIngestionStatus("REPO-CLONE-FAILED");
ksGitInfoRepository.save(ksGitInfo);
resultDTO.setSuccess(false);
resultDTO.setMessage(repoName + " With Branch " + branchName + " clone Failed. "+errorMessage);
simpMessagingTemplate.convertAndSend("/topic/clone-status",resultDTO);
} }
return false; });
} catch (IOException e) {
throw new RuntimeException(e);
}
} }
public GitPullOutput pullChanges(String repoName,String branchName) { public GitPullOutput pullChanges(String repoName,String branchName) {
String localPath= basePath; File repoDirectory = new File(basePath,repoName);
File repoDirectory = new File(localPath,repoName);
GitPullOutput gitPullOutput =new GitPullOutput(); GitPullOutput gitPullOutput =new GitPullOutput();
Map<String,String> gitdiff=null; Map<String,String> gitdiff=null;
try(Git git = Git.open(repoDirectory)) { try(Git git = Git.open(repoDirectory)) {
@@ -236,15 +286,5 @@ public class GitService {
} }
} }
// Method to extract the protocol part
private static String getProtocol(String url) {
int protocolEndIndex = url.indexOf("://");
return protocolEndIndex != -1 ? url.substring(0, protocolEndIndex + 3) : "";
}
// Method to extract the domain part
private static String getDomain(String url) {
int protocolEndIndex = url.indexOf("://");
return protocolEndIndex != -1 ? url.substring(protocolEndIndex + 3) : url;
}
} }

32
src/main/java/com/olympus/apollo/utils/GitUrlUtils.java Normal file
View File

@@ -0,0 +1,32 @@
package com.olympus.apollo.utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.net.MalformedURLException;
import java.net.URL;
public class GitUrlUtils {
public static final Logger logger = LoggerFactory.getLogger(GitUrlUtils.class);
// Method to extract the protocol part
public static String getProtocol(String url) {
try{
URL urlObject = new URL(url);
return urlObject.getProtocol() +"://";
} catch (MalformedURLException e){
logger.error("Invalid URL for protocol extraction: {}",url,e);
return "";
}
}
// Method to extract the domain part
public static String getDomain(String url) {
try{
URL urlObject = new URL(url);
return urlObject.getHost();
} catch (MalformedURLException e) {
logger.error("Invalid URL for domain extraction: {}", url, e);
return url; // Returning the original URL if the domain extraction fails
}
}
}

85
src/main/java/com/olympus/apollo/utils/GitUtils.java Normal file
View File

@@ -0,0 +1,85 @@
package com.olympus.apollo.utils;
import com.olympus.apollo.dto.GitCloneInput;
import com.olympus.apollo.models.KSGitInfo;
import com.olympus.apollo.models.KSGitIngestionInfo;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
public class GitUtils {
private static final Logger logger = LoggerFactory.getLogger(GitUtils.class);
public static KSGitInfo createKSGitInfo(GitCloneInput gitCloneInput,String basePath){
logger.debug("Creating KSGitInfo with repoName: {}, branch: {}",gitCloneInput.getRepoName(),gitCloneInput.getBranch());
KSGitInfo info = new KSGitInfo();
info.setRepoName(gitCloneInput.getRepoName());
info.setBranch(gitCloneInput.getBranch());
info.setCommitId(gitCloneInput.getCommitId());
info.setRepoPath(basePath+ File.separator +gitCloneInput.getBranch());
info.setIngestionStatus("REPO-NEW");
info.setIngestionDate(new Date());
info.setIngestionDateFormat(new SimpleDateFormat("MM/dd/yy").format(new Date()));
logger.info("KSGitInfo Created: {}, {}",info.getRepoName(),info.getBranch());
return info;
}
public static KSGitIngestionInfo createKSGitIngestionInfo(GitCloneInput gitCloneInput){
logger.debug("Creating KSGitIngestionInfo with repoName: {}, branch: {}",gitCloneInput.getRepoName(),gitCloneInput.getBranch());
HashMap<String,String> metaData = createMetaData(gitCloneInput.getRepoName(), gitCloneInput.getBranch());
KSGitIngestionInfo ksGitIngestionInfo = new KSGitIngestionInfo();
ksGitIngestionInfo.setMetadata(metaData);
ksGitIngestionInfo.setMinChunkSizeToEmbed(gitCloneInput.getMinChunkSizeToEmbed());
ksGitIngestionInfo.setMaxNumberOfChunks(gitCloneInput.getMaxNumberOfChunks());
ksGitIngestionInfo.setMinChunkSize(gitCloneInput.getMinChunkSize());
ksGitIngestionInfo.setDefaultChunkSize(gitCloneInput.getDefaultChunkSize());
logger.info("ksGitIngestionInfo Created: {}, {}",gitCloneInput.getRepoName(),gitCloneInput.getBranch());
return ksGitIngestionInfo;
}
private static HashMap<String,String> createMetaData(String repoName, String branchName){
logger.debug("Creating metadata for repoName: {}, branchName: {}", repoName, branchName);
HashMap<String,String> metaData =new HashMap<>();
metaData.put("KsApplicationName", repoName);
metaData.put("KsDoctype", "gitrepository");
metaData.put("KsDocSource", "gitlab");
metaData.put("KsFileSource", repoName);
metaData.put("KsBranch", branchName);
metaData.put("KsRepoName", repoName);
logger.info("Metadata created: {}", metaData);
return metaData;
}
public static boolean branchExists(String branchName, Repository repository){
try{
List<Ref> refs = repository.getRefDatabase().getRefs();
for(Ref ref : refs){
if(ref.getName().equals("refs/heads/"+branchName)){
return true;
}
}
return false;
} catch (IOException e) {
logger.error("Error checking if branch exists: {}", e.getMessage(), e);
throw new RuntimeException("Error accessing the repository", e);
}
}
}