admin/apollo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

clone repo & delete git records websocket done GitService&DeletionService

Browse Source
This commit is contained in:
sumedh
2024-09-12 23:08:12 +05:30
parent 1679ab8dba
commit 8ef069eca8
21 changed files with 700 additions and 328 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.gitignore vendored
View File

@@ -33,8 +33,6 @@ build/
.vscode/
##changes in filepath before deploy
src/main/java/com/olympus/apollo/services/StorageProperties.java
src/main/resources/application.properties
src/main/java/com/olympus/apollo/services/GitService.java
src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java
#src/main/java/com/olympus/apollo/services/StorageProperties.java
#src/main/resources/application.properties

5
pom.xml
View File

@@ -37,6 +37,11 @@
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-websocket</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-mongodb</artifactId>

2
src/main/java/com/olympus/apollo/ApolloApplication.java
View File

@@ -5,7 +5,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.scheduling.annotation.EnableAsync;
import com.olympus.apollo.services.StorageProperties;
import com.olympus.apollo.properties.StorageProperties;
@SpringBootApplication
@EnableConfigurationProperties(StorageProperties.class)

25
src/main/java/com/olympus/apollo/config/WebSocketConfig.java Normal file
View File

@@ -0,0 +1,25 @@
package com.olympus.apollo.config;
import org.springframework.context.annotation.Configuration;
import org.springframework.messaging.simp.config.MessageBrokerRegistry;
import org.springframework.web.socket.config.annotation.EnableWebSocketMessageBroker;
import org.springframework.web.socket.config.annotation.StompEndpointRegistry;
import org.springframework.web.socket.config.annotation.WebSocketMessageBrokerConfigurer;
@Configuration
@EnableWebSocketMessageBroker
public class WebSocketConfig implements WebSocketMessageBrokerConfigurer {
@Override
public void registerStompEndpoints(StompEndpointRegistry registry) {
registry.addEndpoint("/ws/endpoint")
.setAllowedOrigins("http://apollo.olympusai.live")
.withSockJS();
}
@Override
public void configureMessageBroker(MessageBrokerRegistry config) {
config.enableSimpleBroker("/topic"); //should be used as prefix while sending request from spring to browser
config.setApplicationDestinationPrefixes("/app"); //should be used as prefix from browser to spring server
}
}

152
src/main/java/com/olympus/apollo/controllers/FeApi/KSGitController.java
View File

@@ -5,12 +5,19 @@ import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import com.olympus.apollo.dto.*;
import com.olympus.apollo.exception.GitCloneException;
import com.olympus.apollo.services.GitService;
import com.olympus.apollo.utils.GitUtils;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
import org.springframework.http.HttpStatusCode;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
@@ -24,94 +31,79 @@ import com.olympus.apollo.services.KSGitInfoService;
@RequestMapping("/fe-api/ks_git_repos")
public class KSGitController {
@Autowired
private KSGitInfoRepository ksGitInfoRepository;
@Autowired
private KSGitIngestionInfoRepository ksGitIngestionInfoRepository;
@Autowired
private KSGitInfoRepository ksGitInfoRepository;
@Autowired
private KSGitIngestionInfoRepository ksGitIngestionInfoRepository;
@Autowired
private GitService gitService;
@Autowired
private GitService gitService;
@Value("${gitlab.path}")
private String basePath;
@Value("${gitlab.path}")
private String basePath;
@GetMapping("")
public List<KSGitInfo> listGitInfo() {
List<KSGitInfo> result = (List<KSGitInfo>) ksGitInfoRepository.findAll();
return result;
}
private static final Logger logger = LoggerFactory.getLogger(KSGitController.class);
@PostMapping("/uploadRepo")
public ResponseEntity<String> handleGitUpload(@RequestBody KSGitUploadDTO ksGitUploadDTO) {
@GetMapping("")
public List<KSGitInfo> listGitInfo() {
List<KSGitInfo> result = (List<KSGitInfo>) ksGitInfoRepository.findAll();
return result;
}
KSGitInfo ksGitInfo = new KSGitInfo();
ksGitInfo.setRepoName(ksGitUploadDTO.getRepoName());
ksGitInfo.setBranch(ksGitUploadDTO.getBranch());
ksGitInfo.setCommitId(ksGitUploadDTO.getCommitId());
ksGitInfo.setRepoPath(ksGitUploadDTO.getRepoPath());
ksGitInfo.setIngestionStatus("NEW");
ksGitInfo.setIngestionDate(new Date());
ksGitInfo.setIngestionDateFormat(new SimpleDateFormat("MM/dd/yy").format(new Date()));
@PostMapping("/uploadRepo")
public ResponseEntity<String> handleGitUpload(@RequestBody KSGitUploadDTO ksGitUploadDTO) {
KSGitIngestionInfo ksGitIngestionInfo = new KSGitIngestionInfo();
HashMap<String, String> metadata = new HashMap<>();
metadata.put("KsApplicationName", ksGitUploadDTO.getRepoName());
metadata.put("KsDoctype", "gitrepository");
metadata.put("KsDocSource", "gitlab");
metadata.put("KsFileSource", ksGitUploadDTO.getRepoName());
metadata.put("KsBranch", ksGitUploadDTO.getBranch());
metadata.put("KsRepoName", ksGitUploadDTO.getRepoName());
KSGitInfo ksGitInfo = new KSGitInfo();
ksGitInfo.setRepoName(ksGitUploadDTO.getRepoName());
ksGitInfo.setBranch(ksGitUploadDTO.getBranch());
ksGitInfo.setCommitId(ksGitUploadDTO.getCommitId());
ksGitInfo.setRepoPath(ksGitUploadDTO.getRepoPath());
ksGitInfo.setIngestionStatus("NEW");
ksGitInfo.setIngestionDate(new Date());
ksGitInfo.setIngestionDateFormat(new SimpleDateFormat("MM/dd/yy").format(new Date()));
ksGitIngestionInfo.setMetadata(metadata);
ksGitIngestionInfo.setMinChunkSizeToEmbed(ksGitUploadDTO.getMinChunkSizeToEmbed());
ksGitIngestionInfo.setMaxNumberOfChunks(ksGitUploadDTO.getMaxNumberOfChunks());
ksGitIngestionInfo.setMinChunkSize(ksGitUploadDTO.getMinChunkSize());
ksGitIngestionInfo.setDefaultChunkSize(ksGitUploadDTO.getDefaultChunkSize());
KSGitIngestionInfo ksGitIngestionInfo = new KSGitIngestionInfo();
HashMap<String, String> metadata = new HashMap<>();
metadata.put("KsApplicationName", ksGitUploadDTO.getRepoName());
metadata.put("KsDoctype", "gitrepository");
metadata.put("KsDocSource", "gitlab");
metadata.put("KsFileSource", ksGitUploadDTO.getRepoName());
metadata.put("KsBranch", ksGitUploadDTO.getBranch());
metadata.put("KsRepoName", ksGitUploadDTO.getRepoName());
ksGitIngestionInfo.setMetadata(metadata);
ksGitIngestionInfo.setMinChunkSizeToEmbed(ksGitUploadDTO.getMinChunkSizeToEmbed());
ksGitIngestionInfo.setMaxNumberOfChunks(ksGitUploadDTO.getMaxNumberOfChunks());
ksGitIngestionInfo.setMinChunkSize(ksGitUploadDTO.getMinChunkSize());
ksGitIngestionInfo.setDefaultChunkSize(ksGitUploadDTO.getDefaultChunkSize());
ksGitIngestionInfoRepository.save(ksGitIngestionInfo);
ksGitInfo.setKsGitIngestionInfo(ksGitIngestionInfo);
ksGitInfoRepository.save(ksGitInfo);
return ResponseEntity.ok("Upload successful");
}
//clone the repository
@PostMapping("/clone")
public ResponseEntity<ResultDTO> gitClone(@RequestBody GitCloneInput gitCloneInput) throws GitCloneException {
KSGitInfo ksGitInfo = GitUtils.createKSGitInfo(gitCloneInput, basePath);
KSGitIngestionInfo ksGitIngestionInfo = GitUtils.createKSGitIngestionInfo(gitCloneInput);
ksGitIngestionInfoRepository.save(ksGitIngestionInfo);
ksGitInfo.setKsGitIngestionInfo(ksGitIngestionInfo);
ksGitInfoRepository.save(ksGitInfo);
return ResponseEntity.ok("Upload successful");
}
// Start the async processing
gitService.cloneRepository(gitCloneInput.getSource(), gitCloneInput.getRepoName(), gitCloneInput.getBranch(), gitCloneInput.getGroup(), gitCloneInput.getTokenType(), ksGitInfo);
//clone the master branch from remote repository
@PostMapping("/clone")
public GitCloneOutput gitClone(@RequestBody GitCloneInput gitCloneInput) throws GitAPIException, IOException {
KSGitInfo ksGitInfo = new KSGitInfo();
ksGitInfo.setRepoName(gitCloneInput.getRepoName());
ksGitInfo.setBranch(gitCloneInput.getBranch());
ksGitInfo.setCommitId(gitCloneInput.getCommitId());
ksGitInfo.setRepoPath(basePath+"/"+gitCloneInput.getBranch());
ksGitInfo.setIngestionStatus("NEW");
ksGitInfo.setIngestionDate(new Date());
ksGitInfo.setIngestionDateFormat(new SimpleDateFormat("MM/dd/yy").format(new Date()));
KSGitIngestionInfo ksGitIngestionInfo = new KSGitIngestionInfo();
HashMap<String, String> metadata = new HashMap<>();
metadata.put("KsApplicationName", gitCloneInput.getRepoName());
metadata.put("KsDoctype", "gitrepository");
metadata.put("KsDocSource", "gitlab");
metadata.put("KsFileSource", gitCloneInput.getRepoName());
metadata.put("KsBranch", gitCloneInput.getBranch());
metadata.put("KsRepoName", gitCloneInput.getRepoName());
ksGitIngestionInfo.setMetadata(metadata);
ksGitIngestionInfo.setMinChunkSizeToEmbed(gitCloneInput.getMinChunkSizeToEmbed());
ksGitIngestionInfo.setMaxNumberOfChunks(gitCloneInput.getMaxNumberOfChunks());
ksGitIngestionInfo.setMinChunkSize(gitCloneInput.getMinChunkSize());
ksGitIngestionInfo.setDefaultChunkSize(gitCloneInput.getDefaultChunkSize());
ksGitIngestionInfoRepository.save(ksGitIngestionInfo);
ksGitInfo.setKsGitIngestionInfo(ksGitIngestionInfo);
ksGitInfoRepository.save(ksGitInfo);
return gitService.cloneRepository(gitCloneInput.getSource(),gitCloneInput.getRepoName(),gitCloneInput.getBranch(),gitCloneInput.getGroup(),gitCloneInput.getTokenType());
}
//Return an immediate response
ResultDTO response = new ResultDTO();
response.setMessage("Ingestion process initiated for repo: " + gitCloneInput.getRepoName() + " branch: " + gitCloneInput.getBranch());
response.setSuccess(true);
return ResponseEntity.ok(response);
}
/*
curl --location 'http://localhost:8082/fe-api/ks_git_repos/clone' \
--header 'Content-Type: application/json' \
@@ -120,10 +112,10 @@ public class KSGitController {
'
*/
//pull latest changes from master branch
@GetMapping("/pullchanges")
public GitPullOutput gitPull(@RequestParam String repoName,@RequestParam String branchName){
return gitService.pullChanges(repoName,branchName);
}
//pull latest changes from master branch
@GetMapping("/pullchanges")
public GitPullOutput gitPull(@RequestParam String repoName, @RequestParam String branchName) {
return gitService.pullChanges(repoName, branchName);
}
}

2
src/main/java/com/olympus/apollo/controllers/KSFileController.java
View File

@@ -13,7 +13,7 @@ import com.olympus.apollo.models.KSDocument;
import com.olympus.apollo.models.KSIngestionInfo;
import com.olympus.apollo.repository.KSDocumentRepository;
import com.olympus.apollo.repository.KSIngestionInfoRepository;
import com.olympus.apollo.services.StorageFileNotFoundException;
import com.olympus.apollo.exception.StorageFileNotFoundException;
import com.olympus.apollo.services.StorageService;
import com.olympus.apollo.dto.FileUploadDTO;

20
src/main/java/com/olympus/apollo/controllers/TestController.java
View File

@@ -1,7 +1,9 @@
package com.olympus.apollo.controllers;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import com.olympus.apollo.dto.ResultDTO;
import com.olympus.apollo.services.GitService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -55,17 +57,15 @@ public class TestController {
}
@GetMapping("test/ingest_repo")
public ResponseEntity<String> ingestRepo(@RequestParam String repoName,@RequestParam String branchName) {
try {
gitRepositoryIngestor.ingestGitRepository(repoName,branchName);
return ResponseEntity.ok("Ingestion Started");
} catch (Exception e) {
public ResponseEntity<ResultDTO> ingestRepo(@RequestParam String repoName, @RequestParam String branchName) {
// Start the async processing
gitRepositoryIngestor.ingestGitRepository(repoName,branchName);
logger.error("Error during ingestion start", e);
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
.body("Error starting ingestion: " + e.getMessage());
}
//Return an immediate response
ResultDTO response = new ResultDTO();
response.setMessage("Ingestion process initiated for repo: " + repoName + " branch: " + branchName);
response.setSuccess(true);
return ResponseEntity.ok(response);
}
@GetMapping("test/reingest_repo")

10
src/main/java/com/olympus/apollo/dto/ResultDTO.java Normal file
View File

@@ -0,0 +1,10 @@
package com.olympus.apollo.dto;
import lombok.Getter;
import lombok.Setter;
@Setter @Getter
public class ResultDTO {
private boolean success;
private String message;
}

11
src/main/java/com/olympus/apollo/exception/BranchCheckoutException.java Normal file
View File

@@ -0,0 +1,11 @@
package com.olympus.apollo.exception;
public class BranchCheckoutException extends Exception{
public BranchCheckoutException(String message){
super(message);
}
public BranchCheckoutException(String message,Throwable cause){
super(message,cause);
}
}

11
src/main/java/com/olympus/apollo/exception/GitCloneException.java Normal file
View File

@@ -0,0 +1,11 @@
package com.olympus.apollo.exception;
public class GitCloneException extends Exception{
public GitCloneException(String message){
super(message);
}
public GitCloneException(String message,Throwable cause){
super(message,cause);
}
}

25
src/main/java/com/olympus/apollo/exception/GlobalExceptionHandler.java Normal file
View File

@@ -0,0 +1,25 @@
package com.olympus.apollo.exception;
import com.olympus.apollo.dto.GitCloneOutput;
import org.slf4j.LoggerFactory;
import org.slf4j.Logger;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.ControllerAdvice;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.ResponseBody;
@ControllerAdvice
public class GlobalExceptionHandler {
private static final Logger logger = LoggerFactory.getLogger(GlobalExceptionHandler.class);
@ExceptionHandler(GitCloneException.class)
@ResponseBody
public ResponseEntity<GitCloneOutput> handleGitCloneException(GitCloneException ex) {
logger.error("Git clone exception: {}", ex.getMessage(), ex);
GitCloneOutput output = new GitCloneOutput();
output.setMessage("Git clone error: " + ex.getMessage());
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(output);
}
}

2
src/main/java/com/olympus/apollo/services/StorageException.java → src/main/java/com/olympus/apollo/exception/StorageException.java
View File

@@ -1,4 +1,4 @@
package com.olympus.apollo.services;
package com.olympus.apollo.exception;
public class StorageException extends RuntimeException {
public StorageException(String message) {

2
src/main/java/com/olympus/apollo/services/StorageFileNotFoundException.java → src/main/java/com/olympus/apollo/exception/StorageFileNotFoundException.java
View File

@@ -1,4 +1,4 @@
package com.olympus.apollo.services;
package com.olympus.apollo.exception;
public class StorageFileNotFoundException extends StorageException {

10
src/main/java/com/olympus/apollo/exception/vectorStoreMetaDetailsEmptyException.java Normal file
View File

@@ -0,0 +1,10 @@
package com.olympus.apollo.exception;
public class vectorStoreMetaDetailsEmptyException extends RuntimeException{
public vectorStoreMetaDetailsEmptyException(String message){
super(message);
}
public vectorStoreMetaDetailsEmptyException(String message,Throwable cause){
super(message,cause);
}
}

3
src/main/java/com/olympus/apollo/services/StorageProperties.java → src/main/java/com/olympus/apollo/properties/StorageProperties.java
View File

@@ -1,7 +1,6 @@
package com.olympus.apollo.services;
package com.olympus.apollo.properties;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.beans.factory.annotation.Value;
@ConfigurationProperties("storage")
public class StorageProperties {

204
src/main/java/com/olympus/apollo/services/DeletionService.java
View File

@@ -1,6 +1,8 @@
package com.olympus.apollo.services;
import com.olympus.apollo.dto.DeleteGitRepoDetailsRequest;
import com.olympus.apollo.dto.ResultDTO;
import com.olympus.apollo.exception.vectorStoreMetaDetailsEmptyException;
import com.olympus.apollo.models.KSGitInfo;
import com.olympus.apollo.models.KSGitIngestionInfo;
import com.olympus.apollo.models.KSIngestionInfo;
@@ -12,11 +14,13 @@ import com.olympus.apollo.dto.DeletionRequest;
import com.olympus.apollo.dto.VectorStoreMetadataDetails;
import com.olympus.apollo.models.VectorStore;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.messaging.simp.SimpMessagingTemplate;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
@Service
public class DeletionService {
@@ -38,6 +42,9 @@ public class DeletionService {
@Autowired
private VectorStoreRepository vectorStoreRepository;
@Autowired
private SimpMessagingTemplate simpMessagingTemplate;
@Async("asyncTaskExecutor")
public void deleteRecords(DeletionRequest deletionRequest) {
try {
@@ -79,80 +86,143 @@ public class DeletionService {
throw new RuntimeException("An error occurred while deleting records", e);
}
}
@Async("asyncTaskExecutor")
public void deleteRecordsOfGitRepo(DeleteGitRepoDetailsRequest deleteGitRepoDetailsRequest) {
try {
boolean KSGitInfoExists = deleteGitRepoDetailsRequest.getKsGitInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitInfoId().isEmpty() && ksGitInfoRepository.existsById(deleteGitRepoDetailsRequest.getKsGitInfoId());
logger.info("KSGitInfo with id {} exists.", deleteGitRepoDetailsRequest.getKsGitInfoId());
boolean KSGitIngestionInfoExists = deleteGitRepoDetailsRequest.getKsGitIngestionInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitIngestionInfoId().isEmpty() && ksGitIngestionInfoRepository.existsById(deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
logger.info("KSGitIngestionInfo with id {} exists.", deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
boolean vectorStoreGitDetailsExists = deleteGitRepoDetailsRequest.getKsApplicationName() != null && deleteGitRepoDetailsRequest.getKsDocSource() != null && deleteGitRepoDetailsRequest.getKsFileSource() != null && deleteGitRepoDetailsRequest.getKsDoctype() != null && deleteGitRepoDetailsRequest.getKsBranch() != null;
@Async
public CompletableFuture<Void> deleteRecordsOfGitRepo(DeleteGitRepoDetailsRequest deleteGitRepoDetailsRequest) {
return CompletableFuture.runAsync(() -> {
ResultDTO resultDTO = new ResultDTO();
try {
String ksGitInfoId = deleteGitRepoDetailsRequest.getKsGitInfoId();
String ksGitIngestionInfoId = deleteGitRepoDetailsRequest.getKsGitIngestionInfoId();
String applicationName=deleteGitRepoDetailsRequest.getKsApplicationName();
String ksDocSource=deleteGitRepoDetailsRequest.getKsDocSource();
String ksFileSource=deleteGitRepoDetailsRequest.getKsFileSource();
String ksDoctype=deleteGitRepoDetailsRequest.getKsDoctype();
String ksBranch=deleteGitRepoDetailsRequest.getKsBranch();
Optional<KSGitInfo> ksGitInfo = ksGitInfoRepository.findById(deleteGitRepoDetailsRequest.getKsGitInfoId());
String ingestionStatus = ksGitInfo.get().getIngestionStatus();
logger.info("Ingestion Status is {}.",ingestionStatus+" "+Thread.currentThread().getName());
List<VectorStore> vectorStoreMetadataDetails = vectorStoreGitDetailsExists ? vectorStoreRepository.findGitVectorByMetadata(deleteGitRepoDetailsRequest.getKsDoctype(), deleteGitRepoDetailsRequest.getKsDocSource(), deleteGitRepoDetailsRequest.getKsFileSource(), deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsBranch()) : List.of();
boolean KSGitInfoExists = ksGitInfoId != null && !ksGitInfoId.isEmpty() && ksGitInfoRepository.existsById(ksGitInfoId);
boolean KSGitIngestionInfoExists = ksGitIngestionInfoId != null && !ksGitIngestionInfoId.isEmpty() && ksGitIngestionInfoRepository.existsById(ksGitIngestionInfoId);
boolean vectorStoreGitDetailsExists = applicationName != null && ksDocSource != null && ksFileSource != null && ksDoctype != null && ksBranch != null;
if (KSGitInfoExists && KSGitIngestionInfoExists) {
if(ingestionStatus.equals("ERROR")){
if (deleteGitRepoDetailsRequest.getKsGitInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitInfoId().isEmpty()) {
ksGitInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitInfoId());
logger.info("KsGitInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitInfoId());
}
logger.info("KSGitInfo with id {} exists: {}", ksGitInfoId,KSGitInfoExists);
logger.info("KSGitIngestionInfo with id {} exists: {}", ksGitIngestionInfoId,KSGitIngestionInfoExists);
if (deleteGitRepoDetailsRequest.getKsGitIngestionInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitIngestionInfoId().isEmpty()) {
ksGitIngestionInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
logger.info("KSGitIngestionInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
}
if( !vectorStoreMetadataDetails.isEmpty()){
for (VectorStore store : vectorStoreMetadataDetails) {
vectorStoreRepository.deleteById(store.getId());
logger.info("VectorStore with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), store.getId());
}
}
Optional<KSGitInfo> ksGitInfoOpt = ksGitInfoRepository.findById(ksGitInfoId);
if(ksGitInfoOpt.isEmpty()){
String message = applicationName + " With Branch " + ksBranch + " record deletion failed due to KSGitInfo with id "+ksGitInfoId+" does not exist.";
logger.warn(message);
} else if (ingestionStatus.equals("INGESTED") && !vectorStoreMetadataDetails.isEmpty()) {
if (deleteGitRepoDetailsRequest.getKsGitInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitInfoId().isEmpty()) {
ksGitInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitInfoId());
logger.info("KsGitInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitInfoId());
}
if (deleteGitRepoDetailsRequest.getKsGitIngestionInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitIngestionInfoId().isEmpty()) {
ksGitIngestionInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
logger.info("KSGitIngestionInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
}
for (VectorStore store : vectorStoreMetadataDetails) {
vectorStoreRepository.deleteById(store.getId());
logger.info("VectorStore with id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), store.getId());
}
logger.info("All records deleted successfully.");
}else if (ingestionStatus.equals("NEW") && vectorStoreMetadataDetails.isEmpty()) {
if (deleteGitRepoDetailsRequest.getKsGitInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitInfoId().isEmpty()) {
ksGitInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitInfoId());
logger.info("KsGitInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitInfoId());
}
if (deleteGitRepoDetailsRequest.getKsGitIngestionInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitIngestionInfoId().isEmpty()) {
ksGitIngestionInfoRepository.deleteById(deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
logger.info("KSGitIngestionInfo with reponame {} and id {} deleted successfully.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
}
logger.info("records deleted successfully.");
}
} else {
if (!KSGitInfoExists) {
logger.warn("getKsGitInfo with reponame {} and id {} does not exist.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitInfoId());
} else if (!KSGitIngestionInfoExists) {
logger.warn("KSGitIngestionInfo with reponame {} and id {} does not exist.", deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsGitIngestionInfoId());
} else if (vectorStoreMetadataDetails.isEmpty()) {
logger.warn("No VectorStore Data available");
resultDTO.setSuccess(false);
resultDTO.setMessage(message);
simpMessagingTemplate.convertAndSend("/topic/deletion-status",resultDTO);
return;
}
KSGitInfo ksGitInfo = ksGitInfoOpt.get();
String ingestionStatus = ksGitInfo.getIngestionStatus();
logger.info("Ingestion Status is {}.", ingestionStatus);
List<VectorStore> vectorStoreMetadataDetails = vectorStoreGitDetailsExists
? vectorStoreRepository.findGitVectorByMetadata(ksDoctype,ksDocSource, ksFileSource, applicationName, ksBranch)
: List.of();
if (KSGitInfoExists && KSGitIngestionInfoExists) {
deleteRecordsBasedOnIngestionStatus(ksGitInfoId,ksBranch,ingestionStatus,ksGitIngestionInfoId,vectorStoreMetadataDetails,applicationName);
String message = applicationName + " With Branch " + ksBranch + " records removed successfully having KSGitInfo with id "+ksGitInfoId;
logger.info(message);
resultDTO.setSuccess(true);
resultDTO.setMessage(applicationName + " With Branch " + ksBranch + " records removed successfully ");
simpMessagingTemplate.convertAndSend("/topic/deletion-status",resultDTO);
} else {
if (!KSGitInfoExists) {
String message = applicationName + " With Branch " + ksBranch + " record deletion failed due to KSGitInfo with id "+ksGitInfoId+" does not exist.";
logger.error(message);
resultDTO.setSuccess(false);
resultDTO.setMessage(message);
simpMessagingTemplate.convertAndSend("/topic/deletion-status",resultDTO);
} else if (!KSGitIngestionInfoExists) {
String message = applicationName + " With Branch " + ksBranch + " record deletion failed due to KSGitIngestionInfo with id "+ksGitIngestionInfoId+" does not exist.";
logger.error(message);
resultDTO.setSuccess(false);
resultDTO.setMessage(message);
simpMessagingTemplate.convertAndSend("/topic/deletion-status",resultDTO);
} else if (vectorStoreMetadataDetails.isEmpty()) {
String message = applicationName + " With Branch " + ksBranch + " record deletion failed due to No VectorStore Data available";
logger.error(message);
resultDTO.setSuccess(false);
resultDTO.setMessage(message);
simpMessagingTemplate.convertAndSend("/topic/deletion-status",resultDTO);
}
}
} catch (Exception e) {
String message = "An error occurred while deleting records";
logger.error(message,e);
resultDTO.setSuccess(false);
resultDTO.setMessage(message);
simpMessagingTemplate.convertAndSend("/topic/deletion-status",resultDTO);
throw new RuntimeException("An error occurred while deleting records", e);
}
});
}
private void deleteRecordsBasedOnIngestionStatus(String ksGitInfoId,String ksBranch,String ingestionStatus,String ksGitIngestionInfoId,List<VectorStore> vectorStoreMetaDetails,String applicationName){
try{
switch (ingestionStatus){
case "INGESTION-ERROR":
case "INGESTION-IN-PROGRESS":
case "INGESTED":
deleteGitInfoAndIngestionInfo(ksGitInfoId,ksGitIngestionInfoId,applicationName);
deleteVectorStores(vectorStoreMetaDetails,applicationName);
break;
case "REPO-NEW":
case "REPO-CLONE-IN-PROGRESS":
case "REPO-CLONE-COMPLETED":
case "REPO-CLONE-FAILED":
if (vectorStoreMetaDetails.isEmpty()) {
deleteGitInfoAndIngestionInfo(ksGitInfoId, ksGitIngestionInfoId, applicationName);
}else {
// Throw a custom exception if vectorStoreMetaDetails is not empty
throw new vectorStoreMetaDetailsEmptyException("VectorStoreMetaDetails is not empty for application name "+applicationName+" branch "+ksBranch+" and ingestion status is " + ingestionStatus);
}
break;
default:
logger.warn("Unknown ingestion status: {}", ingestionStatus);
}
} catch (vectorStoreMetaDetailsEmptyException e){
logger.error("vectorStoreMetaDetailsEmptyException occurred: ", e);
throw e;
} catch (Exception e){
logger.error("An error occurred while deleting records based on ingestion status: ", e);
throw new RuntimeException("An error occurred while deleting records based on ingestion status", e);
}
}
private void deleteGitInfoAndIngestionInfo(String ksGitInfoId,String ksGitIngestionInfoID,String applicationName){
if (ksGitInfoId != null && !ksGitInfoId.isEmpty()) {
ksGitInfoRepository.deleteById(ksGitInfoId);
logger.info("KsGitInfo with reponame {} and id {} deleted successfully.", applicationName, ksGitInfoId);
}
if (ksGitIngestionInfoID != null && !ksGitIngestionInfoID.isEmpty()) {
ksGitIngestionInfoRepository.deleteById(ksGitIngestionInfoID);
logger.info("KSGitIngestionInfo with reponame {} and id {} deleted successfully.", applicationName, ksGitIngestionInfoID);
}
}
private void deleteVectorStores(List<VectorStore> vectorStoreMetadataDetails, String applicationName){
if(!vectorStoreMetadataDetails.isEmpty()){
for (VectorStore store : vectorStoreMetadataDetails) {
String storeId=store.getId();
vectorStoreRepository.deleteById(storeId);
logger.info("VectorStore with id {} deleted successfully.", applicationName, storeId);
}
} catch (Exception e) {
logger.error("An error occurred while deleting records: ", e);
throw new RuntimeException("An error occurred while deleting records", e);
}
}
}

10
src/main/java/com/olympus/apollo/services/FileSystemStorageService.java
View File

@@ -7,13 +7,11 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.olympus.apollo.exception.StorageException;
import com.olympus.apollo.exception.StorageFileNotFoundException;
import com.olympus.apollo.properties.StorageProperties;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.Resource;
import org.springframework.core.io.UrlResource;
@@ -30,7 +28,7 @@ public class FileSystemStorageService implements StorageService {
public FileSystemStorageService(StorageProperties properties) {
if(properties.getLocation().trim().length() == 0){
throw new StorageException("File upload location can not be Empty.");
throw new StorageException("File upload location can not be Empty.");
}
this.rootLocation = Paths.get(properties.getLocation());

173
src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java
View File

@@ -13,6 +13,10 @@ import java.util.concurrent.CompletableFuture;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.olympus.apollo.dto.GitCloneOutput;
import com.olympus.apollo.dto.ResultDTO;
import com.olympus.apollo.exception.BranchCheckoutException;
import com.olympus.apollo.exception.GitCloneException;
import com.olympus.apollo.repository.VectorStoreRepository;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException;
@@ -26,6 +30,8 @@ import org.springframework.ai.transformer.splitter.TokenTextSplitter;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.ResponseEntity;
import org.springframework.messaging.simp.SimpMessagingTemplate;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
@@ -51,6 +57,9 @@ public class GitRepositoryIngestor {
@Autowired
private GitService gitService;
@Autowired
private SimpMessagingTemplate simpMessagingTemplate;
public GitRepositoryIngestor(VectorStore vectorStore) {
this.vectorStore = vectorStore;
}
@@ -58,84 +67,137 @@ public class GitRepositoryIngestor {
Logger logger = LoggerFactory.getLogger(GitRepositoryIngestor.class);
@Async
public CompletableFuture<Void> ingestGitRepository(String repo,String branchName) {
Optional<KSGitInfo> optionalDocument = ksGitInfoRepository.findByRepoNameAndBranchName(repo,branchName);
if (optionalDocument.isPresent()) {
KSGitInfo ksGitInfo = optionalDocument.get();
if ("NEW".equals(ksGitInfo.getIngestionStatus()) || "ERROR".equals(ksGitInfo.getIngestionStatus())) {
ingestRepo(repo,branchName, ksGitInfo);
} else {
logger.info("OOPS: Document is already Injected");
public CompletableFuture<Void> ingestGitRepository(String repo, String branchName) {
return CompletableFuture.runAsync(() -> {
try {
Optional<KSGitInfo> optionalDocument = ksGitInfoRepository.findByRepoNameAndBranchName(repo,branchName);
if (optionalDocument.isPresent()) {
KSGitInfo ksGitInfo = optionalDocument.get();
if ("REPO-CLONE-COMPLETED".equals(ksGitInfo.getIngestionStatus())) {
ResultDTO repoResult = ingestRepo(repo,branchName, ksGitInfo);
// Notify clients via WebSocket or other mechanisms
//notifyClients(repo, branchName, repoResult);
} else {
String message = "Document is already ingested or has a different status: " + ksGitInfo.getIngestionStatus();
logger.info(message);
}
} else {
String message = "Document not found for repo: " + repo + " branch: " + branchName;
logger.info(message);
}
} catch (Exception e){
String errorMessage = "UnExpected Error: "+ e.getMessage();
logger.error(errorMessage,e);
// Handle exception and notify clients
//notifyClients(repo, branchName, new ResultDTO(false, errorMessage));
}
} else {
logger.info("OOPS: Document Not found");
}
return CompletableFuture.completedFuture(null);
});
}
private void ingestRepo(String repoName,String branchName, KSGitInfo ksGitInfo) {
String repoPath = basePath+"/"+ repoName + "/";
//String repoPath = basePath + "\\" + repoName + "\\"; //need to modify before deploy
gitService.checkOutRepository(repoName,branchName);
logger.info("Repository path : " + repoPath);
try (Git git = Git.open(new File(repoPath))) {
ksGitInfo.setIngestionStatus("IN PROGRESS");
private ResultDTO ingestRepo(String repoName,String branchName, KSGitInfo ksGitInfo) {
KSGitIngestionInfo ingestionInfo = ksGitInfo.getKsGitIngestionInfo();
logger.info("Metadata : " + ingestionInfo.getMetadata());
ksGitInfoRepository.save(ksGitInfo);
String repoPath = basePath + File.separator + repoName;
logger.info("Starting ingestion for repo: {} and branch: {}", repoName, branchName);
ResultDTO resultDTO = new ResultDTO();
Repository repository = git.getRepository();
RevCommit latestCommit = git.log().setMaxCount(1).call().iterator().next();
try {
gitService.checkOutRepository(repoName, branchName);
logger.info("Repository path of Ingestion : {}", repoPath);
try (TreeWalk treeWalk = new TreeWalk(repository)) {
treeWalk.addTree(latestCommit.getTree());
treeWalk.setRecursive(true);
try (Git git = Git.open(new File(repoPath))) {
ksGitInfo.setIngestionStatus("INGESTION-IN-PROGRESS");
ksGitInfoRepository.save(ksGitInfo);
List<Document> documents = new ArrayList<>();
KSGitIngestionInfo ingestionInfo = ksGitInfo.getKsGitIngestionInfo();
logger.info("Metadata of ingestionInfo: {}",ingestionInfo.getMetadata());
while (treeWalk.next()) {
String filePath = treeWalk.getPathString();
String fileName = treeWalk.getNameString();
Repository repository = git.getRepository();
RevCommit latestCommit = git.log().setMaxCount(1).call().iterator().next();
if (isRelevantFile(fileName)) {
byte[] fileContent = repository.open(treeWalk.getObjectId(0)).getBytes();
String fileContentStr = new String(fileContent, StandardCharsets.UTF_8);
try (TreeWalk treeWalk = new TreeWalk(repository)) {
treeWalk.addTree(latestCommit.getTree());
treeWalk.setRecursive(true);
Map<String, String> metadata = extractMetadata(fileName, fileContentStr);
metadata.put("filePath", filePath);
metadata.put("fileName", fileName);
List<Document> documents = new ArrayList<>();
Document doc = new Document(fileContentStr);
doc.getMetadata().putAll(metadata);
while (treeWalk.next()) {
String filePath = treeWalk.getPathString();
String fileName = treeWalk.getNameString();
doc.getMetadata().putAll(ingestionInfo.getMetadata());
documents.add(doc);
if (isRelevantFile(fileName)) {
byte[] fileContent = repository.open(treeWalk.getObjectId(0)).getBytes();
String fileContentStr = new String(fileContent, StandardCharsets.UTF_8);
Map<String, String> metadata = extractMetadata(fileName, fileContentStr);
metadata.put("filePath", filePath);
metadata.put("fileName", fileName);
Document doc = new Document(fileContentStr);
doc.getMetadata().putAll(metadata);
doc.getMetadata().putAll(ingestionInfo.getMetadata());
documents.add(doc);
}
}
TokenTextSplitter splitter = new TokenTextSplitter(
ingestionInfo.getDefaultChunkSize(),
ingestionInfo.getMinChunkSize(),
ingestionInfo.getMinChunkSizeToEmbed(),
ingestionInfo.getMaxNumberOfChunks(),
false
);
List<Document> splitDocuments = splitter.split(documents);
logger.info("Number of documents to be embedded: {}", splitDocuments.size());
vectorStore.add(splitDocuments);
logger.info("Documents embedded Successfully");
} catch (IOException e) {
ksGitInfo.setIngestionStatus("INGESTION-ERROR");
ksGitInfo.setIngestionDate(new Date());
ksGitInfoRepository.save(ksGitInfo);
logger.error("Error processing repository content", e);
resultDTO.setSuccess(false);
resultDTO.setMessage("Error processing repository content: " + e.getMessage());
return resultDTO;
}
TokenTextSplitter splitter = new TokenTextSplitter(ingestionInfo.getDefaultChunkSize(),
ingestionInfo.getMinChunkSize(), ingestionInfo.getMinChunkSizeToEmbed(),
ingestionInfo.getMaxNumberOfChunks(), false);
ksGitInfo.setIngestionStatus("INGESTED");
ksGitInfo.setIngestionDate(new Date());
ksGitInfoRepository.save(ksGitInfo);
List<Document> splitDocuments = splitter.split(documents);
logger.info("Number of documents: " + splitDocuments.size());
vectorStore.add(splitDocuments);
logger.info("Documents embedded");
resultDTO.setSuccess(true);
resultDTO.setMessage("Ingestion completed successfully");
} catch (IOException e) {
ksGitInfo.setIngestionStatus("INGESTION-ERROR");
ksGitInfoRepository.save(ksGitInfo);
logger.error("Error opening repository", e);
resultDTO.setSuccess(false);
resultDTO.setMessage("Error opening repository: " + e.getMessage());
}
}catch (BranchCheckoutException e){
ksGitInfo.setIngestionStatus("INGESTION-ERROR");
ksGitInfoRepository.save(ksGitInfo);
ksGitInfo.setIngestionStatus("INGESTED");
ksGitInfo.setIngestionDate(new Date());
ksGitInfoRepository.save(ksGitInfo);
logger.error("Error checking out repository branch", e);
resultDTO.setSuccess(false);
resultDTO.setMessage("Error checking out repository branch: " + e.getMessage());
} catch (Exception e) {
ksGitInfo.setIngestionStatus("ERROR");
ksGitInfo.setIngestionStatus("INGESTION-ERROR");
ksGitInfoRepository.save(ksGitInfo);
logger.error("Error during ingestion", e);
resultDTO.setSuccess(false);
resultDTO.setMessage("Unexpected error during ingestion: " + e.getMessage());
}
return resultDTO;
}
public CompletableFuture<Void> ReIngestGitRepository(String repo,String branchName) throws GitAPIException, IOException {
public CompletableFuture<Void> ReIngestGitRepository(String repo,String branchName) throws GitAPIException, IOException, BranchCheckoutException {
Optional<KSGitInfo> optionalDocument = ksGitInfoRepository.findByRepoNameAndBranchName(repo,branchName);
if (optionalDocument.isPresent()) {
KSGitInfo ksGitInfo = optionalDocument.get();
@@ -150,7 +212,7 @@ public class GitRepositoryIngestor {
return CompletableFuture.completedFuture(null);
}
private void reIngestRepo(String repoName,String branchName, KSGitInfo ksGitInfo) throws IOException, GitAPIException {
private void reIngestRepo(String repoName,String branchName, KSGitInfo ksGitInfo) throws IOException, GitAPIException, BranchCheckoutException {
HashMap<String, String> modifiedFiles = ksGitInfo.getGitModifiedFiles();
@@ -192,8 +254,7 @@ public class GitRepositoryIngestor {
}
}
gitService.checkOutRepository(repoName,branchName);
String repoPath = basePath+"/"+ repoName + "/";
//String repoPath = basePath+ "\\" + repoName + "\\"; //need to modify before deploy
String repoPath = basePath+ File.separator + repoName;
logger.info("Repository path : " + repoPath);
try (Git git = Git.open(new File(repoPath))) {

238
src/main/java/com/olympus/apollo/services/GitService.java
View File

@@ -2,33 +2,40 @@ package com.olympus.apollo.services;
import com.olympus.apollo.dto.GitCloneOutput;
import com.olympus.apollo.dto.GitPullOutput;
import com.olympus.apollo.dto.ResultDTO;
import com.olympus.apollo.exception.BranchCheckoutException;
import com.olympus.apollo.exception.GitCloneException;
import com.olympus.apollo.models.KSGitInfo;
import com.olympus.apollo.repository.KSGitInfoRepository;
import com.olympus.apollo.repository.VectorStoreRepository;
import com.olympus.apollo.utils.GitUrlUtils;
import com.olympus.apollo.utils.GitUtils;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.api.errors.RefNotFoundException;
import org.eclipse.jgit.diff.DiffEntry;
import org.eclipse.jgit.diff.DiffFormatter;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.eclipse.jgit.transport.FetchResult;
import org.eclipse.jgit.treewalk.AbstractTreeIterator;
import org.eclipse.jgit.treewalk.CanonicalTreeParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.messaging.simp.SimpMessagingTemplate;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
@Service
public class GitService {
@@ -47,113 +54,156 @@ public class GitService {
private String gitHubToken="23";
public GitCloneOutput cloneRepository(String Source,String repoName,String branchName,String group, String tokenType) throws GitAPIException {
GitCloneOutput gitCloneOutput = new GitCloneOutput();
try {
String gitlabToken;
String remoteRepoUrl;
switch (tokenType) {
case "cloud":
gitlabToken = cloudGitlabToken;
remoteRepoUrl = getProtocol(Source) + "gitlab-ci-token:" + gitlabToken + "@" + getDomain(Source) + "/" + group + "/" + repoName + ".git";
break;
case "onpremises":
gitlabToken = onPremisesGitlabToken;
remoteRepoUrl = getProtocol(Source) + "gitlab-ci-token:" + gitlabToken + "@" + getDomain(Source) + "/" + group + "/" + repoName + ".git";
break;
case "github":
gitlabToken = gitHubToken;
remoteRepoUrl = Source + "/" + group + "/" + repoName + ".git";
break;
default:
gitCloneOutput.setMessage("tokenType is invalid");
gitCloneOutput.setRepoName(repoName);
return gitCloneOutput;
}
System.out.println("remoteUrl : " + remoteRepoUrl);
//System.out.println("gitlab token : " + gitlabToken);
String localPath = basePath;
System.out.println("localpath"+basePath);
File cloneDirectory = new File(localPath,repoName);
//Files.createDirectories(Paths.get(localPath));
//File cloneDirectory = new File(localPath, repoName);
//System.out.println("cloneDirectory : " + cloneDirectory);
if(!cloneDirectory.exists()){
if (tokenType.equals("github")) {
Git.cloneRepository().setURI(remoteRepoUrl).setDirectory(cloneDirectory).call();
} else {
Git.cloneRepository().setURI(remoteRepoUrl).setDirectory(cloneDirectory).setCredentialsProvider(new UsernamePasswordCredentialsProvider("username", gitlabToken)).call();
}
}else {
System.out.println("Directory already exists. Skipping clone.");
}
@Autowired
private SimpMessagingTemplate simpMessagingTemplate;
private static final Logger logger = LoggerFactory.getLogger(GitService.class);
checkOutRepository(repoName, branchName);
gitCloneOutput.setRepoName(repoName);
gitCloneOutput.setMessage(repoName + "With Branch " + branchName + " cloned successfully");
}catch (Exception e){
gitCloneOutput.setRepoName(repoName);
gitCloneOutput.setMessage("Error occured : "+e.getMessage());
}
return gitCloneOutput;
private String buildGitURL(String source,String token,String group,String repoName){
return GitUrlUtils.getProtocol(source) + "gitlab-ci-token:" + token + "@" + GitUrlUtils.getDomain(source) + "/" + group + "/" + repoName + ".git";
}
public String checkOutRepository(String repoName, String branchName) {
String localPath = basePath;
File repoDirectory = new File(localPath,repoName);
private void clone(String tokenType,String remoteRepoUrl,File cloneDirectory,String token) throws GitAPIException{
if ("github".equals(tokenType)) {
Git.cloneRepository()
.setURI(remoteRepoUrl)
.setDirectory(cloneDirectory)
.call();
} else {
Git.cloneRepository()
.setURI(remoteRepoUrl)
.setDirectory(cloneDirectory)
.setCredentialsProvider(new UsernamePasswordCredentialsProvider("username", token))
.call();
}
}
public String checkOutRepository(String repoName, String branchName) throws BranchCheckoutException {
File repoDirectory = new File(basePath,repoName);
try(Git git = Git.open(repoDirectory)) {
Repository repository = git.getRepository();
if(!branchExists(branchName,repository)) {
if(!GitUtils.branchExists(branchName,repository)) {
FetchResult fetchResult = git.fetch().call();
System.out.println("Fetch result: " + fetchResult.getMessages());
git.checkout().setCreateBranch(true).setName(branchName).setStartPoint("origin/"+branchName).call();
logger.info("Fetch result: {}",fetchResult.getMessages());
git.checkout()
.setCreateBranch(true)
.setName(branchName)
.setStartPoint("origin/"+branchName)
.call();
logger.info("Checked out new branch: {}", branchName);
}else {
git.checkout().setName(branchName).setCreateBranch(false).call();
System.out.println("Checked out existing branch: " + branchName);
git.checkout()
.setName(branchName)
.setCreateBranch(false)
.call();
logger.info("Checked out existing branch: {}", branchName);
}
return branchName+" checkout successful";
} catch (RefNotFoundException e) {
try(Git git = Git.open(repoDirectory)) {
git.checkout().setName("origin/"+branchName).setCreateBranch(false).call();
return branchName+" checkout successful";
} catch (RefNotFoundException ex) {
return "checkout failed with error "+e.getMessage();
}catch (Exception ex2){
System.out.println("An error occurred: "+ ex2.getMessage());
return "checkout failed with error "+ ex2.getMessage();
}
String errorMessage = "Branch Not Found: " + e.getMessage();
logger.error(errorMessage, e);
throw new BranchCheckoutException(errorMessage, e);
} catch (Exception e){
System.out.println("An error occurred: "+ e.getMessage());
return "checkout failed with error "+e.getMessage();
String errorMessage = "Checkout Failed with an error: " + e.getMessage();
logger.error(errorMessage, e);
throw new BranchCheckoutException("Checkout failed with error: " + e.getMessage(), e);
}
}
private static boolean branchExists(String branchName,Repository repository){
try{
List<Ref> refs = repository.getRefDatabase().getRefs();
@Async
public CompletableFuture<Void> cloneRepository(String Source, String repoName, String branchName, String group, String tokenType,KSGitInfo ksGitInfo) {
return CompletableFuture.runAsync(() -> {
ksGitInfo.setIngestionStatus("REPO-CLONE-IN-PROGRESS");
ksGitInfoRepository.save(ksGitInfo);
for(Ref ref : refs){
System.out.println("refs "+ref.getName());
}
ResultDTO resultDTO = new ResultDTO();
try {
String gitlabToken;
String remoteRepoUrl;
switch (tokenType) {
case "cloud":
gitlabToken = cloudGitlabToken;
remoteRepoUrl = buildGitURL(Source,gitlabToken,group,repoName);
break;
case "onpremises":
gitlabToken = onPremisesGitlabToken;
remoteRepoUrl = buildGitURL(Source,gitlabToken,group,repoName);
break;
case "github":
gitlabToken = gitHubToken;
remoteRepoUrl = Source + "/" + group + "/" + repoName + ".git";
break;
default:
String errorMessage = repoName + "With Branch " + branchName + " having invalid tokenType. ";
logger.error(errorMessage);
for(Ref ref : refs){
if(ref.getName().equals("refs/heads/"+branchName)){
return true;
throw new GitCloneException(errorMessage);
}
logger.info("Remote URL: {}", remoteRepoUrl);
File cloneDirectory = new File(basePath,repoName);
if(!cloneDirectory.exists()){
clone(tokenType,remoteRepoUrl,cloneDirectory,gitlabToken);
}else {
logger.info("Directory already exists. Skipping clone.");
}
String checkoutMessage = checkOutRepository(repoName, branchName);
ksGitInfo.setIngestionStatus("REPO-CLONE-COMPLETED");
ksGitInfoRepository.save(ksGitInfo);
resultDTO.setSuccess(true);
resultDTO.setMessage(repoName + " With Branch " + branchName + " cloned successfully. "+checkoutMessage);
simpMessagingTemplate.convertAndSend("/topic/clone-status",resultDTO);
}catch (GitAPIException e){
String errorMessage = "Git API error: " + e.getMessage();
logger.error(errorMessage, e);
ksGitInfo.setIngestionStatus("REPO-CLONE-FAILED");
ksGitInfoRepository.save(ksGitInfo);
resultDTO.setSuccess(false);
resultDTO.setMessage(repoName + " With Branch " + branchName + " clone Failed. "+errorMessage);
simpMessagingTemplate.convertAndSend("/topic/clone-status",resultDTO);
} catch (BranchCheckoutException e){
String errorMessage = "Branch Checkout Error: "+ e.getMessage();
logger.error(errorMessage,e);
ksGitInfo.setIngestionStatus("REPO-CLONE-FAILED");
ksGitInfoRepository.save(ksGitInfo);
resultDTO.setSuccess(false);
resultDTO.setMessage(repoName + " With Branch " + branchName + " clone Failed. "+errorMessage);
simpMessagingTemplate.convertAndSend("/topic/clone-status",resultDTO);
} catch (GitCloneException e){
String errorMessage = "Git Clone Error: "+ e.getMessage();
logger.error(errorMessage,e);
ksGitInfo.setIngestionStatus("REPO-CLONE-FAILED");
ksGitInfoRepository.save(ksGitInfo);
resultDTO.setSuccess(false);
resultDTO.setMessage(repoName + " With Branch " + branchName + " clone Failed. "+errorMessage);
simpMessagingTemplate.convertAndSend("/topic/clone-status",resultDTO);
} catch (Exception e){
String errorMessage = "UnExpected Error: "+ e.getMessage();
logger.error(errorMessage,e);
ksGitInfo.setIngestionStatus("REPO-CLONE-FAILED");
ksGitInfoRepository.save(ksGitInfo);
resultDTO.setSuccess(false);
resultDTO.setMessage(repoName + " With Branch " + branchName + " clone Failed. "+errorMessage);
simpMessagingTemplate.convertAndSend("/topic/clone-status",resultDTO);
}
return false;
} catch (IOException e) {
throw new RuntimeException(e);
}
});
}
public GitPullOutput pullChanges(String repoName,String branchName) {
String localPath= basePath;
File repoDirectory = new File(localPath,repoName);
File repoDirectory = new File(basePath,repoName);
GitPullOutput gitPullOutput =new GitPullOutput();
Map<String,String> gitdiff=null;
try(Git git = Git.open(repoDirectory)) {
@@ -236,15 +286,5 @@ public class GitService {
}
}
// Method to extract the protocol part
private static String getProtocol(String url) {
int protocolEndIndex = url.indexOf("://");
return protocolEndIndex != -1 ? url.substring(0, protocolEndIndex + 3) : "";
}
// Method to extract the domain part
private static String getDomain(String url) {
int protocolEndIndex = url.indexOf("://");
return protocolEndIndex != -1 ? url.substring(protocolEndIndex + 3) : url;
}
}

32
src/main/java/com/olympus/apollo/utils/GitUrlUtils.java Normal file
View File

@@ -0,0 +1,32 @@
package com.olympus.apollo.utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.net.MalformedURLException;
import java.net.URL;
public class GitUrlUtils {
public static final Logger logger = LoggerFactory.getLogger(GitUrlUtils.class);
// Method to extract the protocol part
public static String getProtocol(String url) {
try{
URL urlObject = new URL(url);
return urlObject.getProtocol() +"://";
} catch (MalformedURLException e){
logger.error("Invalid URL for protocol extraction: {}",url,e);
return "";
}
}
// Method to extract the domain part
public static String getDomain(String url) {
try{
URL urlObject = new URL(url);
return urlObject.getHost();
} catch (MalformedURLException e) {
logger.error("Invalid URL for domain extraction: {}", url, e);
return url; // Returning the original URL if the domain extraction fails
}
}
}

85
src/main/java/com/olympus/apollo/utils/GitUtils.java Normal file
View File

@@ -0,0 +1,85 @@
package com.olympus.apollo.utils;
import com.olympus.apollo.dto.GitCloneInput;
import com.olympus.apollo.models.KSGitInfo;
import com.olympus.apollo.models.KSGitIngestionInfo;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
public class GitUtils {
private static final Logger logger = LoggerFactory.getLogger(GitUtils.class);
public static KSGitInfo createKSGitInfo(GitCloneInput gitCloneInput,String basePath){
logger.debug("Creating KSGitInfo with repoName: {}, branch: {}",gitCloneInput.getRepoName(),gitCloneInput.getBranch());
KSGitInfo info = new KSGitInfo();
info.setRepoName(gitCloneInput.getRepoName());
info.setBranch(gitCloneInput.getBranch());
info.setCommitId(gitCloneInput.getCommitId());
info.setRepoPath(basePath+ File.separator +gitCloneInput.getBranch());
info.setIngestionStatus("REPO-NEW");
info.setIngestionDate(new Date());
info.setIngestionDateFormat(new SimpleDateFormat("MM/dd/yy").format(new Date()));
logger.info("KSGitInfo Created: {}, {}",info.getRepoName(),info.getBranch());
return info;
}
public static KSGitIngestionInfo createKSGitIngestionInfo(GitCloneInput gitCloneInput){
logger.debug("Creating KSGitIngestionInfo with repoName: {}, branch: {}",gitCloneInput.getRepoName(),gitCloneInput.getBranch());
HashMap<String,String> metaData = createMetaData(gitCloneInput.getRepoName(), gitCloneInput.getBranch());
KSGitIngestionInfo ksGitIngestionInfo = new KSGitIngestionInfo();
ksGitIngestionInfo.setMetadata(metaData);
ksGitIngestionInfo.setMinChunkSizeToEmbed(gitCloneInput.getMinChunkSizeToEmbed());
ksGitIngestionInfo.setMaxNumberOfChunks(gitCloneInput.getMaxNumberOfChunks());
ksGitIngestionInfo.setMinChunkSize(gitCloneInput.getMinChunkSize());
ksGitIngestionInfo.setDefaultChunkSize(gitCloneInput.getDefaultChunkSize());
logger.info("ksGitIngestionInfo Created: {}, {}",gitCloneInput.getRepoName(),gitCloneInput.getBranch());
return ksGitIngestionInfo;
}
private static HashMap<String,String> createMetaData(String repoName, String branchName){
logger.debug("Creating metadata for repoName: {}, branchName: {}", repoName, branchName);
HashMap<String,String> metaData =new HashMap<>();
metaData.put("KsApplicationName", repoName);
metaData.put("KsDoctype", "gitrepository");
metaData.put("KsDocSource", "gitlab");
metaData.put("KsFileSource", repoName);
metaData.put("KsBranch", branchName);
metaData.put("KsRepoName", repoName);
logger.info("Metadata created: {}", metaData);
return metaData;
}
public static boolean branchExists(String branchName, Repository repository){
try{
List<Ref> refs = repository.getRefDatabase().getRefs();
for(Ref ref : refs){
if(ref.getName().equals("refs/heads/"+branchName)){
return true;
}
}
return false;
} catch (IOException e) {
logger.error("Error checking if branch exists: {}", e.getMessage(), e);
throw new RuntimeException("Error accessing the repository", e);
}
}
}