diff --git a/.gitignore b/.gitignore index 549e00a..d2d0fbd 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,10 @@ build/ ### VS Code ### .vscode/ + +##changes in filepath before deploy +src/main/java/com/olympus/apollo/services/StorageProperties.java +src/main/resources/application.properties +src/main/java/com/olympus/apollo/services/GitService.java +src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java + diff --git a/pom.xml b/pom.xml index 36e1e6f..b389881 100644 --- a/pom.xml +++ b/pom.xml @@ -28,6 +28,7 @@ 21 + 1.0.0-M1 @@ -53,13 +54,6 @@ pom - - org.springframework.boot spring-boot-starter-data-rest @@ -141,15 +135,6 @@ never - - spring-snapshots - Spring Snapshots - https://repo.spring.io/snapshot - - false - never - - diff --git a/src/main/java/com/olympus/apollo/config/AsyncConfiguration.java b/src/main/java/com/olympus/apollo/config/AsyncConfiguration.java new file mode 100644 index 0000000..f4229e1 --- /dev/null +++ b/src/main/java/com/olympus/apollo/config/AsyncConfiguration.java @@ -0,0 +1,24 @@ +package com.olympus.apollo.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +import java.util.concurrent.Executor; +import java.util.concurrent.ThreadPoolExecutor; + +@Configuration +@EnableAsync //using this annotation spring boot will internally search which all methods have annotation async +public class AsyncConfiguration { + @Bean("asyncTaskExecutor") + public Executor asyncTaskExecutor(){ + ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor(); + taskExecutor.setCorePoolSize(4); //these threads are kept alive even if they are idle + taskExecutor.setQueueCapacity(150); //maximum number of tasks that can be held in queue if number gets exceeded then it will create new thread as per maximum pool size defined + taskExecutor.setMaxPoolSize(4); + taskExecutor.setThreadNamePrefix("AsyncTaskThread-"); + taskExecutor.initialize(); + return taskExecutor; + } +} diff --git a/src/main/java/com/olympus/apollo/controllers/FeApi/KSGitController.java b/src/main/java/com/olympus/apollo/controllers/FeApi/KSGitController.java index cd0008f..96f7b93 100644 --- a/src/main/java/com/olympus/apollo/controllers/FeApi/KSGitController.java +++ b/src/main/java/com/olympus/apollo/controllers/FeApi/KSGitController.java @@ -10,6 +10,7 @@ import com.olympus.apollo.dto.*; import com.olympus.apollo.services.GitService; import org.eclipse.jgit.api.errors.GitAPIException; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; @@ -19,7 +20,6 @@ import com.olympus.apollo.repository.KSGitInfoRepository; import com.olympus.apollo.repository.KSGitIngestionInfoRepository; import com.olympus.apollo.services.KSGitInfoService; -@CrossOrigin(origins = "http://localhost:5173") @RestController @RequestMapping("/fe-api/ks_git_repos") public class KSGitController { @@ -32,6 +32,9 @@ public class KSGitController { @Autowired private GitService gitService; + @Value("${gitlab.path}") + private String basePath; + @GetMapping("") public List listGitInfo() { List result = (List) ksGitInfoRepository.findAll(); @@ -81,7 +84,7 @@ public class KSGitController { ksGitInfo.setRepoName(gitCloneInput.getRepoName()); ksGitInfo.setBranch(gitCloneInput.getBranch()); ksGitInfo.setCommitId(gitCloneInput.getCommitId()); - ksGitInfo.setRepoPath(gitCloneInput.getRepoPath()); + ksGitInfo.setRepoPath(basePath+"/"+gitCloneInput.getBranch()); ksGitInfo.setIngestionStatus("NEW"); ksGitInfo.setIngestionDate(new Date()); ksGitInfo.setIngestionDateFormat(new SimpleDateFormat("MM/dd/yy").format(new Date())); @@ -107,7 +110,7 @@ public class KSGitController { ksGitInfo.setKsGitIngestionInfo(ksGitIngestionInfo); ksGitInfoRepository.save(ksGitInfo); - return gitService.cloneRepository(gitCloneInput.getSource(),gitCloneInput.getRepoName(),gitCloneInput.getGroup(),gitCloneInput.getTokenType()); + return gitService.cloneRepository(gitCloneInput.getSource(),gitCloneInput.getRepoName(),gitCloneInput.getBranch(),gitCloneInput.getGroup(),gitCloneInput.getTokenType()); } /* curl --location 'http://localhost:8082/fe-api/ks_git_repos/clone' \ @@ -118,9 +121,9 @@ public class KSGitController { */ //pull latest changes from master branch - @GetMapping("/pullchanges/{repoName}") - public GitPullOutput gitPull(@PathVariable String repoName){ - return gitService.pullChanges(repoName); + @GetMapping("/pullchanges") + public GitPullOutput gitPull(@RequestParam String repoName,@RequestParam String branchName){ + return gitService.pullChanges(repoName,branchName); } } diff --git a/src/main/java/com/olympus/apollo/controllers/FeApi/KsDocumentController.java b/src/main/java/com/olympus/apollo/controllers/FeApi/KsDocumentController.java index 0abd934..953e20a 100644 --- a/src/main/java/com/olympus/apollo/controllers/FeApi/KsDocumentController.java +++ b/src/main/java/com/olympus/apollo/controllers/FeApi/KsDocumentController.java @@ -11,7 +11,6 @@ import com.olympus.apollo.repository.KSDocumentRepository; @RestController @RequestMapping("/fe-api/ksdocuments") -@CrossOrigin(origins = "http://localhost:5173") public class KsDocumentController { @Autowired diff --git a/src/main/java/com/olympus/apollo/controllers/FeApi/VectorStoreController.java b/src/main/java/com/olympus/apollo/controllers/FeApi/VectorStoreController.java index 0c831ce..183a0b0 100644 --- a/src/main/java/com/olympus/apollo/controllers/FeApi/VectorStoreController.java +++ b/src/main/java/com/olympus/apollo/controllers/FeApi/VectorStoreController.java @@ -7,13 +7,13 @@ import com.olympus.apollo.models.VectorStore; import com.olympus.apollo.repository.VectorStoreRepository; import com.olympus.apollo.services.DeletionService; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; import java.util.List; @RestController @RequestMapping("/fe-api/vector-store") -@CrossOrigin(origins = "http://localhost:5173") public class VectorStoreController { @Autowired private VectorStoreRepository vectorStoreRepository; @@ -45,15 +45,15 @@ public class VectorStoreController { } @PostMapping("/deleteRecords") - public String deleteRecords(@RequestBody DeletionRequest deletionRequest){ + public ResponseEntity deleteRecords(@RequestBody DeletionRequest deletionRequest){ deletionService.deleteRecords(deletionRequest); - return "Records Deleted Successfully"; + return ResponseEntity.ok("Request In Working"); } @PostMapping("/deleteGitRecords") - public String deleteGitRecords(@RequestBody DeleteGitRepoDetailsRequest deleteGitRepoDetailsRequest){ + public ResponseEntity deleteGitRecords(@RequestBody DeleteGitRepoDetailsRequest deleteGitRepoDetailsRequest){ deletionService.deleteRecordsOfGitRepo(deleteGitRepoDetailsRequest); - return "Git Records Deleted Successfully"; + return ResponseEntity.ok("Git Records Deletion IN PROGRESS"); } } diff --git a/src/main/java/com/olympus/apollo/controllers/KSFileController.java b/src/main/java/com/olympus/apollo/controllers/KSFileController.java index 54c7520..308a6a0 100644 --- a/src/main/java/com/olympus/apollo/controllers/KSFileController.java +++ b/src/main/java/com/olympus/apollo/controllers/KSFileController.java @@ -16,7 +16,7 @@ import com.olympus.apollo.repository.KSIngestionInfoRepository; import com.olympus.apollo.services.StorageFileNotFoundException; import com.olympus.apollo.services.StorageService; import com.olympus.apollo.dto.FileUploadDTO; -@CrossOrigin + @RestController public class KSFileController { @Autowired diff --git a/src/main/java/com/olympus/apollo/controllers/TestController.java b/src/main/java/com/olympus/apollo/controllers/TestController.java index e209545..6415512 100644 --- a/src/main/java/com/olympus/apollo/controllers/TestController.java +++ b/src/main/java/com/olympus/apollo/controllers/TestController.java @@ -54,10 +54,10 @@ public class TestController { return "Deleted"; } - @GetMapping("test/ingest_repo/{repoName}") - public ResponseEntity ingestRepo(@PathVariable String repoName) { + @GetMapping("test/ingest_repo") + public ResponseEntity ingestRepo(@RequestParam String repoName,@RequestParam String branchName) { try { - gitRepositoryIngestor.ingestGitRepository(repoName); + gitRepositoryIngestor.ingestGitRepository(repoName,branchName); return ResponseEntity.ok("Ingestion Started"); } catch (Exception e) { @@ -68,11 +68,11 @@ public class TestController { } } - @GetMapping("test/reingest_repo/{repoName}") - public ResponseEntity ReIngestRepo(@PathVariable String repoName) { + @GetMapping("test/reingest_repo") + public ResponseEntity ReIngestRepo(@RequestParam String repoName,@RequestParam String branchName) { try { - gitService.pullChanges(repoName); - gitRepositoryIngestor.ReIngestGitRepository(repoName); + gitService.pullChanges(repoName,branchName); + gitRepositoryIngestor.ReIngestGitRepository(repoName,branchName); return ResponseEntity.ok("Ingestion Started"); } catch (Exception e) { @@ -82,15 +82,12 @@ public class TestController { .body("Error starting ingestion: " + e.getMessage()); } } - /* - curl --location 'http://localhost:8082/test/reingest_repo/shellExecutionThroughAPI' \ - --header 'Authorization: Bearer ' - */ - @GetMapping("test/check_ingestion_status/{repoName}") - public ResponseEntity checkIngestionStatus(@PathVariable String repoName) { + + @GetMapping("test/check_ingestion_status") + public ResponseEntity checkIngestionStatus(@RequestParam String repoName,@RequestParam String branchName) { try { - IngestionOutput ingestionOutput = gitRepositoryIngestor.checkIngestionStatus(repoName); + IngestionOutput ingestionOutput = gitRepositoryIngestor.checkIngestionStatus(repoName,branchName); return ResponseEntity.ok(ingestionOutput); } catch (Exception e) { diff --git a/src/main/java/com/olympus/apollo/dto/DeleteGitRepoDetailsRequest.java b/src/main/java/com/olympus/apollo/dto/DeleteGitRepoDetailsRequest.java index 1c07751..bcf9c72 100644 --- a/src/main/java/com/olympus/apollo/dto/DeleteGitRepoDetailsRequest.java +++ b/src/main/java/com/olympus/apollo/dto/DeleteGitRepoDetailsRequest.java @@ -11,4 +11,5 @@ public class DeleteGitRepoDetailsRequest { private String ksDocSource; private String ksFileSource; private String ksApplicationName; + private String ksBranch; } diff --git a/src/main/java/com/olympus/apollo/repository/KSGitInfoRepository.java b/src/main/java/com/olympus/apollo/repository/KSGitInfoRepository.java index 0ff2ea7..ffabb51 100644 --- a/src/main/java/com/olympus/apollo/repository/KSGitInfoRepository.java +++ b/src/main/java/com/olympus/apollo/repository/KSGitInfoRepository.java @@ -18,4 +18,7 @@ public interface KSGitInfoRepository extends MongoRepository @Query("{'repoName': ?0, 'ksGitIngestionInfo.metadata.KsApplicationName': ?1}") Optional findByMetadataAndRepoName(String repoName, String KsApplicationName); + + @Query("{'repoName': ?0, 'ksGitIngestionInfo.metadata.KsBranch': ?1}") + Optional findByRepoNameAndBranchName(String repoName, String KsApplicationName); } diff --git a/src/main/java/com/olympus/apollo/repository/VectorStoreRepository.java b/src/main/java/com/olympus/apollo/repository/VectorStoreRepository.java index a999e45..391f60b 100644 --- a/src/main/java/com/olympus/apollo/repository/VectorStoreRepository.java +++ b/src/main/java/com/olympus/apollo/repository/VectorStoreRepository.java @@ -23,8 +23,11 @@ public interface VectorStoreRepository extends MongoRepository findAllDetails(); + @Query("{'metadata.KsDoctype': ?0, 'metadata.KsDocSource': ?1, 'metadata.KsFileSource': ?2, 'metadata.KsApplicationName': ?3, 'metadata.KsBranch': ?4}") + List findGitVectorByMetadata(String ksDoctype, String ksDocSource, String ksFileSource, String ksApplicationName, String ksBranch); + @Query("{'metadata.KsDoctype': ?0, 'metadata.KsDocSource': ?1, 'metadata.KsFileSource': ?2, 'metadata.KsApplicationName': ?3}") - List findByMetadata(String ksDoctype, String ksDocSource, String ksFileSource, String ksApplicationName); + List findDocumentVectorByMetadata(String ksDoctype, String ksDocSource, String ksFileSource, String ksApplicationName); @Query("{'metadata.filePath': ?0}") Optional findByFilePath(String filePath); diff --git a/src/main/java/com/olympus/apollo/services/DeletionService.java b/src/main/java/com/olympus/apollo/services/DeletionService.java index 467712b..ca0c2a5 100644 --- a/src/main/java/com/olympus/apollo/services/DeletionService.java +++ b/src/main/java/com/olympus/apollo/services/DeletionService.java @@ -12,6 +12,7 @@ import com.olympus.apollo.dto.DeletionRequest; import com.olympus.apollo.dto.VectorStoreMetadataDetails; import com.olympus.apollo.models.VectorStore; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import java.util.List; @@ -37,6 +38,7 @@ public class DeletionService { @Autowired private VectorStoreRepository vectorStoreRepository; + @Async("asyncTaskExecutor") public void deleteRecords(DeletionRequest deletionRequest) { try { boolean KSDocumentExists = deletionRequest.getKsDocumentId() != null && !deletionRequest.getKsDocumentId().isEmpty() && ksDocumentRepository.existsById(deletionRequest.getKsDocumentId()); @@ -44,53 +46,53 @@ public class DeletionService { boolean vectorStoreExists = deletionRequest.getKsApplicationName() != null && deletionRequest.getKsDocSource() != null && deletionRequest.getKsFileSource() != null && deletionRequest.getKsDoctype() != null; - List vectorStoreMetadataDetails = vectorStoreExists ? vectorStoreRepository.findByMetadata(deletionRequest.getKsDoctype(), deletionRequest.getKsDocSource(), deletionRequest.getKsFileSource(), deletionRequest.getKsApplicationName()) : List.of(); + List vectorStoreMetadataDetails = vectorStoreExists ? vectorStoreRepository.findDocumentVectorByMetadata(deletionRequest.getKsDoctype(), deletionRequest.getKsDocSource(), deletionRequest.getKsFileSource(), deletionRequest.getKsApplicationName()) : List.of(); if (KSDocumentExists && KSIngestionInfoExists && !vectorStoreMetadataDetails.isEmpty()) { if (deletionRequest.getKsDocumentId() != null && !deletionRequest.getKsDocumentId().isEmpty()) { ksDocumentRepository.deleteById(deletionRequest.getKsDocumentId()); - logger.info("KSDocument with id {} deleted successfully.", deletionRequest.getKsDocumentId()); + logger.info("KSDocument with id {} deleted successfully.", deletionRequest.getKsDocumentId()+" "+Thread.currentThread().getName()); } if (deletionRequest.getKsIngestionInfoId() != null && !deletionRequest.getKsIngestionInfoId().isEmpty()) { ksIngestionInfoRepository.deleteById(deletionRequest.getKsIngestionInfoId()); - logger.info("KSIngestionInfo with id {} deleted successfully.", deletionRequest.getKsIngestionInfoId()); + logger.info("KSIngestionInfo with id {} deleted successfully.", deletionRequest.getKsIngestionInfoId()+" "+Thread.currentThread().getName()); } for (VectorStore store : vectorStoreMetadataDetails) { vectorStoreRepository.deleteById(store.getId()); - logger.info("VectorStore with id {} deleted successfully.", store.getId()); + logger.info("VectorStore with id {} deleted successfully.", store.getId()+" "+Thread.currentThread().getName()); } logger.info("All records deleted successfully."); } else { if (!KSDocumentExists) { - logger.warn("KSDocument with id {} does not exist.", deletionRequest.getKsDocumentId()); + logger.warn("KSDocument with id {} does not exist.", deletionRequest.getKsDocumentId()+" "+Thread.currentThread().getName()); } else if (!KSIngestionInfoExists) { - logger.warn("KSIngestionInfo with id {} does not exist.", deletionRequest.getKsIngestionInfoId()); + logger.warn("KSIngestionInfo with id {} does not exist.", deletionRequest.getKsIngestionInfoId()+" "+Thread.currentThread().getName()); } else if (vectorStoreMetadataDetails.isEmpty()) { - logger.warn("No VectorStore Data available"); + logger.warn("No VectorStore Data available",Thread.currentThread().getName()); } } } catch (Exception e) { - logger.error("An error occurred while deleting records: ", e); + logger.error("An error occurred while deleting records: ", e+" "+Thread.currentThread().getName()); throw new RuntimeException("An error occurred while deleting records", e); } } - + @Async("asyncTaskExecutor") public void deleteRecordsOfGitRepo(DeleteGitRepoDetailsRequest deleteGitRepoDetailsRequest) { try { boolean KSGitInfoExists = deleteGitRepoDetailsRequest.getKsGitInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitInfoId().isEmpty() && ksGitInfoRepository.existsById(deleteGitRepoDetailsRequest.getKsGitInfoId()); logger.info("KSGitInfo with id {} exists.", deleteGitRepoDetailsRequest.getKsGitInfoId()); boolean KSGitIngestionInfoExists = deleteGitRepoDetailsRequest.getKsGitIngestionInfoId() != null && !deleteGitRepoDetailsRequest.getKsGitIngestionInfoId().isEmpty() && ksGitIngestionInfoRepository.existsById(deleteGitRepoDetailsRequest.getKsGitIngestionInfoId()); logger.info("KSGitIngestionInfo with id {} exists.", deleteGitRepoDetailsRequest.getKsGitIngestionInfoId()); - boolean vectorStoreGitDetailsExists = deleteGitRepoDetailsRequest.getKsApplicationName() != null && deleteGitRepoDetailsRequest.getKsDocSource() != null && deleteGitRepoDetailsRequest.getKsFileSource() != null && deleteGitRepoDetailsRequest.getKsDoctype() != null; + boolean vectorStoreGitDetailsExists = deleteGitRepoDetailsRequest.getKsApplicationName() != null && deleteGitRepoDetailsRequest.getKsDocSource() != null && deleteGitRepoDetailsRequest.getKsFileSource() != null && deleteGitRepoDetailsRequest.getKsDoctype() != null && deleteGitRepoDetailsRequest.getKsBranch() != null; Optional ksGitInfo = ksGitInfoRepository.findById(deleteGitRepoDetailsRequest.getKsGitInfoId()); String ingestionStatus = ksGitInfo.get().getIngestionStatus(); - logger.info("Ingestion Status is {}.",ingestionStatus); - List vectorStoreMetadataDetails = vectorStoreGitDetailsExists ? vectorStoreRepository.findByMetadata(deleteGitRepoDetailsRequest.getKsDoctype(), deleteGitRepoDetailsRequest.getKsDocSource(), deleteGitRepoDetailsRequest.getKsFileSource(), deleteGitRepoDetailsRequest.getKsApplicationName()) : List.of(); + logger.info("Ingestion Status is {}.",ingestionStatus+" "+Thread.currentThread().getName()); + List vectorStoreMetadataDetails = vectorStoreGitDetailsExists ? vectorStoreRepository.findGitVectorByMetadata(deleteGitRepoDetailsRequest.getKsDoctype(), deleteGitRepoDetailsRequest.getKsDocSource(), deleteGitRepoDetailsRequest.getKsFileSource(), deleteGitRepoDetailsRequest.getKsApplicationName(), deleteGitRepoDetailsRequest.getKsBranch()) : List.of(); if (KSGitInfoExists && KSGitIngestionInfoExists) { if(ingestionStatus.equals("ERROR")){ diff --git a/src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java b/src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java index 7d7c5d7..d515cf6 100644 --- a/src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java +++ b/src/main/java/com/olympus/apollo/services/GitRepositoryIngestor.java @@ -40,7 +40,7 @@ public class GitRepositoryIngestor { private final VectorStore vectorStore; @Value("${gitlab.path}") - private String localRepoPath; + private String basePath; @Autowired private KSGitInfoRepository ksGitInfoRepository; @@ -55,16 +55,12 @@ public class GitRepositoryIngestor { Logger logger = LoggerFactory.getLogger(GitRepositoryIngestor.class); @Async - public CompletableFuture ingestGitRepository(String repo) { - //String repoPath = "C:\\Users\\s.shamrao.shinde\\GenAIStorage\\" + repo + "\\"; - //String repoPath = ksGitInfo.getRepoPath() + repo + "\\"; - //logger.info("Repository path : " + repoPath); - - Optional optionalDocument = ksGitInfoRepository.findByRepoName(repo); + public CompletableFuture ingestGitRepository(String repo,String branchName) { + Optional optionalDocument = ksGitInfoRepository.findByRepoNameAndBranchName(repo,branchName); if (optionalDocument.isPresent()) { KSGitInfo ksGitInfo = optionalDocument.get(); if ("NEW".equals(ksGitInfo.getIngestionStatus()) || "ERROR".equals(ksGitInfo.getIngestionStatus())) { - ingestRepo(repo, ksGitInfo); + ingestRepo(repo,branchName, ksGitInfo); } else { logger.info("OOPS: Document is already Injected"); } @@ -75,9 +71,9 @@ public class GitRepositoryIngestor { } - private void ingestRepo(String repo, KSGitInfo ksGitInfo) { - String repoPath = localRepoPath+"/"+ repo + "/"; - //String repoPath = "C:\\repos\\olympus_ai\\gitClone" + "\\" + repo + "\\"; + private void ingestRepo(String repo,String branchName, KSGitInfo ksGitInfo) { + String repoPath = basePath+"/"+branchName+"/"+ repo + "/"; + //String repoPath = basePath + "\\" + branchName + "\\" + repo + "\\"; //need to modify before deploy logger.info("Repository path : " + repoPath); try (Git git = Git.open(new File(repoPath))) { ksGitInfo.setIngestionStatus("IN PROGRESS"); @@ -135,18 +131,14 @@ public class GitRepositoryIngestor { } } - public CompletableFuture ReIngestGitRepository(String repo) throws GitAPIException, IOException { - //String repoPath = "C:\\Users\\s.shamrao.shinde\\GenAIStorage\\" + repo + "\\"; - //String repoPath = ksGitInfo.getRepoPath() + repo + "\\"; - //logger.info("Repository path : " + repoPath); - + public CompletableFuture ReIngestGitRepository(String repo,String branchName) throws GitAPIException, IOException { Optional optionalDocument = ksGitInfoRepository.findByRepoName(repo); if (optionalDocument.isPresent()) { KSGitInfo ksGitInfo = optionalDocument.get(); if ("INGESTED".equals(ksGitInfo.getIngestionStatus())) { - reIngestRepo(repo, ksGitInfo); + reIngestRepo(repo, branchName,ksGitInfo); } else { - logger.info("OOPS: Document is already Injected"); + logger.info("OOPS: Document is not igested"); } } else { logger.info("OOPS: Document Not found"); @@ -154,7 +146,7 @@ public class GitRepositoryIngestor { return CompletableFuture.completedFuture(null); } - private void reIngestRepo(String repo, KSGitInfo ksGitInfo) throws IOException, GitAPIException { + private void reIngestRepo(String repo,String branchName, KSGitInfo ksGitInfo) throws IOException, GitAPIException { HashMap modifiedFiles = ksGitInfo.getGitModifiedFiles(); @@ -194,10 +186,9 @@ public class GitRepositoryIngestor { vectorStoreRepository.deleteById(vectorStoreId); } } - } - String repoPath = localRepoPath +"/"+ repo + "/"; - //String repoPath = "C:\\repos\\olympus_ai\\gitClone" + "\\" + repo + "\\"; //need to modify before deploy + String repoPath = basePath+"/"+branchName +"/"+ repo + "/"; + //String repoPath = basePath+ "\\" + branchName + "\\" + repo + "\\"; //need to modify before deploy logger.info("Repository path : " + repoPath); try (Git git = Git.open(new File(repoPath))) { @@ -270,8 +261,8 @@ public class GitRepositoryIngestor { } } - public IngestionOutput checkIngestionStatus(String repoName) { - Optional optionalDocument = ksGitInfoRepository.findByRepoName(repoName); + public IngestionOutput checkIngestionStatus(String repoName,String branchName) { + Optional optionalDocument = ksGitInfoRepository.findByRepoNameAndBranchName(repoName,branchName); IngestionOutput ingestionOutput = new IngestionOutput(); if (optionalDocument.isPresent()) { KSGitInfo ksGitInfo = optionalDocument.get(); diff --git a/src/main/java/com/olympus/apollo/services/GitService.java b/src/main/java/com/olympus/apollo/services/GitService.java index b39c72c..b935aff 100644 --- a/src/main/java/com/olympus/apollo/services/GitService.java +++ b/src/main/java/com/olympus/apollo/services/GitService.java @@ -21,6 +21,8 @@ import org.springframework.stereotype.Service; import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -33,7 +35,7 @@ public class GitService { private KSGitInfoRepository ksGitInfoRepository; @Value("${gitlab.path}") - private String localPath; + private String basePath; @Value("${gitlab.cloud.token}") private String cloudGitlabToken; @@ -43,49 +45,57 @@ public class GitService { private String gitHubToken="23"; - public GitCloneOutput cloneRepository(String Source,String repoName,String group, String tokenType) throws GitAPIException { - GitCloneOutput gitCloneOutput= new GitCloneOutput(); - String gitlabToken; - String remoteRepoUrl; - switch (tokenType){ - case "cloud": - gitlabToken=cloudGitlabToken; - remoteRepoUrl= getProtocol(Source)+"gitlab-ci-token:"+gitlabToken+"@"+getDomain(Source)+"/"+group+"/"+repoName+".git"; - break; - case "onpremises": - gitlabToken=onPremisesGitlabToken; - remoteRepoUrl= getProtocol(Source)+"gitlab-ci-token:"+gitlabToken+"@"+getDomain(Source)+"/"+group+"/"+repoName+".git"; - break; - case "github": - gitlabToken=gitHubToken; - remoteRepoUrl= Source+"/"+group+"/"+repoName+".git"; - //String remoteRepoUrl= "https://github.com/automationtester23/shellExecutionThroughAPI.git"; - break; - default: - gitCloneOutput.setMessage("tokenType is invalid"); - gitCloneOutput.setRepoName(repoName); - return gitCloneOutput; + public GitCloneOutput cloneRepository(String Source,String repoName,String branchName,String group, String tokenType) throws GitAPIException { + GitCloneOutput gitCloneOutput = new GitCloneOutput(); + try { + String gitlabToken; + String remoteRepoUrl; + switch (tokenType) { + case "cloud": + gitlabToken = cloudGitlabToken; + remoteRepoUrl = getProtocol(Source) + "gitlab-ci-token:" + gitlabToken + "@" + getDomain(Source) + "/" + group + "/" + repoName + ".git"; + break; + case "onpremises": + gitlabToken = onPremisesGitlabToken; + remoteRepoUrl = getProtocol(Source) + "gitlab-ci-token:" + gitlabToken + "@" + getDomain(Source) + "/" + group + "/" + repoName + ".git"; + break; + case "github": + gitlabToken = gitHubToken; + remoteRepoUrl = Source + "/" + group + "/" + repoName + ".git"; + break; + default: + gitCloneOutput.setMessage("tokenType is invalid"); + gitCloneOutput.setRepoName(repoName); + return gitCloneOutput; + } + System.out.println("remoteUrl : " + remoteRepoUrl); + System.out.println("gitlab token : " + gitlabToken); + String localPath = basePath + "/" + branchName; + //String localPath = basePath + "\\" + branchName; //need to change before deploy + Files.createDirectories(Paths.get(localPath)); + File cloneDirectory = new File(localPath, repoName); + System.out.println("cloneDirectory : " + cloneDirectory); + if (tokenType.equals("github")) { + Git.cloneRepository().setURI(remoteRepoUrl).setDirectory(cloneDirectory).call(); + } else { + Git.cloneRepository().setURI(remoteRepoUrl).setDirectory(cloneDirectory).setCredentialsProvider(new UsernamePasswordCredentialsProvider("username", gitlabToken)).call(); + } + + checkOutRepository(repoName, branchName); + + gitCloneOutput.setRepoName(repoName); + gitCloneOutput.setMessage(repoName + "With Branch " + branchName + " cloned successfully"); + + }catch (Exception e){ + gitCloneOutput.setRepoName(repoName); + gitCloneOutput.setMessage("Error occured : "+e.getMessage()); } - System.out.println("remoteUrl : "+remoteRepoUrl); - System.out.println("gitlab token : "+gitlabToken); - - File cloneDirectory = new File(localPath,repoName); - System.out.println("cloneDirectory : "+cloneDirectory); - if(tokenType.equals("github")){ - Git.cloneRepository().setURI(remoteRepoUrl).setDirectory(cloneDirectory).call(); - }else { - Git.cloneRepository().setURI(remoteRepoUrl).setDirectory(cloneDirectory).setCredentialsProvider(new UsernamePasswordCredentialsProvider("username",gitlabToken)).call(); - } - - checkOutRepository(repoName,"master"); - - gitCloneOutput.setRepoName(repoName); - gitCloneOutput.setMessage(repoName +" cloned successfully"); - return gitCloneOutput; } public String checkOutRepository(String repoName, String branchName) { + String localPath = basePath+"/"+branchName; + //String localPath = basePath+"\\"+branchName; //need to change before deploy File repoDirectory = new File(localPath,repoName); try(Git git = Git.open(repoDirectory)) { git.checkout().setName(branchName).setCreateBranch(false).call(); @@ -106,15 +116,17 @@ public class GitService { } } - public GitPullOutput pullChanges(String repoName) { + public GitPullOutput pullChanges(String repoName,String branchName) { + String localPath= basePath+"/"+branchName; + //String localPath= basePath+"\\"+branchName; //need ot change before deploy File repoDirectory = new File(localPath,repoName); GitPullOutput gitPullOutput =new GitPullOutput(); Map gitdiff=null; try(Git git = Git.open(repoDirectory)) { - String message = checkOutRepository(repoName,"master"); + String message = checkOutRepository(repoName,branchName); System.out.println("message from checkoutRepository : "+message); - gitdiff = getModifiedFiles(git); + gitdiff = getModifiedFiles(git,branchName); HashMap modifiedFiles = new HashMap<>(); @@ -155,12 +167,12 @@ public class GitService { return gitPullOutput; } - private Map getModifiedFiles(Git git) throws GitAPIException, IOException { + private Map getModifiedFiles(Git git,String branchName) throws GitAPIException, IOException { Repository repository = git.getRepository(); git.fetch().call(); AbstractTreeIterator oldTreeParser = prepareTreeParser(repository, "HEAD"); - AbstractTreeIterator newTreeParser = prepareTreeParser(repository, "refs/remotes/origin/master"); + AbstractTreeIterator newTreeParser = prepareTreeParser(repository, "refs/remotes/origin/"+branchName); Map modifiedFiles = new HashMap<>(); try (DiffFormatter formatter = new DiffFormatter(System.out)) { diff --git a/src/main/java/com/olympus/apollo/services/StorageProperties.java b/src/main/java/com/olympus/apollo/services/StorageProperties.java index 5023301..3bc1c71 100644 --- a/src/main/java/com/olympus/apollo/services/StorageProperties.java +++ b/src/main/java/com/olympus/apollo/services/StorageProperties.java @@ -11,6 +11,7 @@ public class StorageProperties { //private String location = ingestionRepositoryBasePath; private String location = "/mnt/apollo_storage/documents"; + //private String location = "C:\\repos\\olympus_ai\\Documents"; public String getLocation() { return location; } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 570840b..ecd45cc 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -32,4 +32,4 @@ ingestion.repository.basepath=C:\\Users\\andrea.terzani\\dev\\Olympus gitlab.token= #gitlab.path=C:\\repos\\olympus_ai\\gitClone -gitlab.path=/mnt/apollo_storage/repository; \ No newline at end of file +gitlab.path=/mnt/apollo_storage/repository \ No newline at end of file